GitHubivarvong

ply

main
ply6 git calls379 ms26 KiB3 cachedprotocol v2
134 lines7.9 KiBRaw
1// Dynamic-block code sets against zlib: ply's inflate accepts a header if and only if node:zlib (inflateRawSync) accepts
2// it, and when both accept the bytes are equal. zlib refuses an over-subscribed set always and an incomplete one except
3// the cases inftrees.c permits (no codes at all; one 1-bit code in the literal/length or distance set). The expected
4// value of every case here is zlib's answer on the same bytes, never ply's: the corpus is every length assignment over
5// a small alphabet for each of the three sets (so every over-subscribed and incomplete shape of that size), then
6// random sets.
7import { test } from 'vitest';
8import assert from 'node:assert/strict';
9import zlib from 'node:zlib';
10import { Rng } from './fuzz/prng.js';
11import { inflate } from '../src/inflate.js';
13// RFC 1951 §3.2.7: the order the code-length code lengths are sent in
14const ORDER = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15];
15// RFC 1951 §3.2.2: canonical codes from lengths (bl_count, next_code); null for a symbol with no code
16function canon(lengths) {
17 const count = new Array(17).fill(0);
18 for (const l of lengths) count[l] += 1;
19 count[0] = 0;
20 const next = new Array(17).fill(0);
21 let code = 0;
22 for (let b = 1; b <= 15; b++) { code = (code + count[b - 1]) << 1; next[b] = code; }
23 return lengths.map(l => (l ? { code: next[l]++, len: l } : null));
25function bitWriter() {
26 const bits = [];
27 return {
28 put(v, n) { for (let i = 0; i < n; i++) bits.push((v >> i) & 1); }, // a field: least significant bit first
29 code(c) { for (let i = c.len - 1; i >= 0; i--) bits.push((c.code >> i) & 1); }, // a Huffman code: most significant first
30 bytes() { const out = new Uint8Array(Math.ceil(bits.length / 8)); bits.forEach((b, i) => { out[i >> 3] |= b << (i & 7); }); return out; },
31 };
33// One final dynamic block. clen: the 19 code-length code lengths by symbol; lit (257..286 lengths) and dist (1..30) are
34// sent one code-length symbol each. A length clen has no code for ends the header there; `tail` follows either way.
35function dynBlock(clen, lit, dist, body, tail = new Uint8Array(0)) {
36 const w = bitWriter();
37 w.put(1, 1); w.put(2, 2);
38 w.put(lit.length - 257, 5); w.put(dist.length - 1, 5); w.put(19 - 4, 4);
39 for (const s of ORDER) w.put(clen[s], 3);
40 const cl = canon(clen);
41 const sent = [...lit, ...dist];
42 if (sent.every(l => cl[l] !== null)) {
43 for (const l of sent) w.code(cl[l]);
44 body(w, canon(lit), canon(dist));
45 }
46 return Uint8Array.from([...w.bytes(), ...tail]);
48// lengths 0..15 as 4-bit codes, no repeat symbols: a complete code-length code (16 × 2^-4 = 1)
49const PLAIN = Array.from({ length: 19 }, (_, s) => (s < 16 ? 4 : 0));
50const lengthsOf = (n, set) => { const a = new Array(n).fill(0); for (const [s, l] of Object.entries(set)) a[Number(s)] = l; return a; };
52const SETS = ['invalid code lengths set', 'invalid literal/lengths set', 'invalid distances set'];
53const seen = { accepted: 0, refused: 0, 'invalid code lengths set': 0, 'invalid literal/lengths set': 0, 'invalid distances set': 0 };
54function agrees(buf, label) {
55 let want = null;
56 let why = '';
57 try { want = new Uint8Array(zlib.inflateRawSync(buf)); } catch (e) { why = e.message; }
58 const got = inflate(buf, 0, { raw: true });
59 const hex = Buffer.from(buf.subarray(0, 48)).toString('hex');
60 if (want === null) {
61 seen.refused += 1;
62 if (SETS.includes(why)) seen[why] += 1;
63 assert.equal(got.ok, false, `${label}: zlib refuses (${why}), ply accepts ${got.ok ? got.value.out.length : 0} bytes: ${hex}`);
64 assert.equal(got.error.code, 'pack_corrupt', label);
65 // the three code-set refusals carry zlib's own message
66 if (SETS.includes(why)) assert.equal(got.error.message, `inflate: ${why}`, `${label}: ${hex}`);
67 return;
68 }
69 seen.accepted += 1;
70 assert.equal(got.ok, true, `${label}: zlib accepts ${want.length} bytes, ply refuses (${got.ok ? '' : got.error.message}): ${hex}`);
71 assert.deepEqual(new Uint8Array(got.value.out), want, `${label}: bytes`);
74test('literal/length sets: every assignment of lengths 0..4 to three literals and 1..4 to end-of-block, as zlib judges it', () => {
75 for (let a = 0; a <= 4; a++) for (let b = 0; b <= 4; b++) for (let c = 0; c <= 4; c++) for (let e = 1; e <= 4; e++) {
76 const lit = lengthsOf(257, { 97: a, 98: b, 99: c, 256: e });
77 const buf = dynBlock(PLAIN, lit, [1], (w, L) => { for (const s of [97, 98, 99, 97]) if (L[s]) w.code(L[s]); w.code(L[256]); });
78 agrees(buf, `lit a:${a} b:${b} c:${c} eob:${e}`);
79 }
80 // both kinds of refusal and the complete sets were all in the corpus (zlib's count, not ply's)
81 assert.ok(seen.accepted >= 10 && seen['invalid literal/lengths set'] >= 100, JSON.stringify(seen));
82});
84test('distance sets: every assignment of lengths 0..3 to three distance codes, with and without a match, as zlib judges it', () => {
85 const before = seen['invalid distances set'];
86 const lit = lengthsOf(258, { 97: 1, 256: 2, 257: 2 }); // complete: 1/2 + 1/4 + 1/4; 257 is a match of length 3
87 for (let x = 0; x <= 3; x++) for (let y = 0; y <= 3; y++) for (let z = 0; z <= 3; z++) for (const match of [false, true]) {
88 const buf = dynBlock(PLAIN, lit, [x, y, z], (w, L, D) => {
89 w.code(L[97]); w.code(L[97]); w.code(L[97]);
90 if (match) { w.code(L[257]); if (D[0]) w.code(D[0]); else w.put(0, 2); } // distance code 0 is distance 1
91 w.code(L[256]);
92 });
93 agrees(buf, `dist ${x},${y},${z} match:${match}`);
94 }
95 assert.ok(seen['invalid distances set'] - before >= 20, JSON.stringify(seen));
96});
98test('code-length sets: every assignment of lengths 0..3 to the code-length symbols 0, 1, 2 and 18, as zlib judges it', () => {
99 const before = seen['invalid code lengths set'];
100 const lit = lengthsOf(258, { 97: 1, 256: 2, 257: 2 });
101 const rng = new Rng(11);
102 for (let p = 0; p <= 3; p++) for (let q = 0; q <= 3; q++) for (let r = 0; r <= 3; r++) for (let t = 0; t <= 3; t++) {
103 const clen = lengthsOf(19, { 0: p, 1: q, 2: r, 18: t });
104 const buf = dynBlock(clen, lit, [1], (w, L) => { w.code(L[97]); w.code(L[97]); w.code(L[256]); }, rng.bytes(4));
105 agrees(buf, `clen 0:${p} 1:${q} 2:${r} 18:${t}`);
106 }
107 assert.ok(seen['invalid code lengths set'] - before >= 100, JSON.stringify(seen));
108});
110test('random code sets: complete codes with one length moved, and random headers, as zlib judges them', () => {
111 const rng = new Rng(20261111);
112 for (let i = 0; i < 600; i++) {
113 // a complete code built by splitting leaves, then (two times in three) one length moved by one: incomplete or over-subscribed
114 const grow = n => { const ls = [1, 1]; while (ls.length < n) { const k = rng.int(0, ls.length - 1); if (ls[k] >= 15) break; ls[k] += 1; ls.push(ls[k]); } return ls; };
115 const bend = ls => { const k = rng.int(0, ls.length - 1); const m = rng.int(0, 2); if (m === 1 && ls[k] < 15) ls[k] += 1; if (m === 2 && ls[k] > 1) ls[k] -= 1; return ls; };
116 const ll = bend(grow(rng.int(2, 40)));
117 const lit = new Array(257 + rng.int(0, 29)).fill(0);
118 const live = [256];
119 lit[256] = ll[0];
120 for (const l of ll.slice(1)) { const s = rng.int(0, 255); if (lit[s] === 0) { lit[s] = l; live.push(s); } }
121 const dl = bend(grow(rng.int(2, 30)));
122 const dist = rng.int(0, 3) === 0 ? [rng.int(0, 1)] : dl;
123 const buf = dynBlock(PLAIN, lit, dist, (w, L) => { for (let j = rng.int(0, 30); j > 0; j--) w.code(L[rng.pick(live.filter(s => s !== 256).concat(live.length > 1 ? [] : [256]))]); w.code(L[256]); });
124 agrees(buf, `random set ${i}`);
125 }
126 for (let i = 0; i < 1500; i++) {
127 // a dynamic block header and code-length code lengths biased short, then random bytes
128 const clen = Array.from({ length: 19 }, () => rng.pick([0, 0, 1, 2, 2, 3, 3, 4, 5, 7]));
129 const w = bitWriter();
130 w.put(1, 1); w.put(2, 2); w.put(rng.int(0, 29), 5); w.put(rng.int(0, 29), 5); w.put(rng.int(0, 15), 4);
131 for (const s of ORDER) w.put(clen[s], 3);
132 agrees(Uint8Array.from([...w.bytes(), ...rng.bytes(rng.int(0, 120))]), `random header ${i}`);
133 }
134});
Diagnostics6 git calls · 379 ms
CallStatusSizeTime0379 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB94 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=94
git.response.bytes=11990
git.response.cache=EXPIRED
git.response.duration_ms=94
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2003.1 KiB276 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=276
git.response.bytes=3125
git.response.cache=MISS
git.response.duration_ms=276
git.fetch.wants=c204e78985eda1da8e528397221f7d623af454bf
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3125
git.pkt.payload.max=3098
git.sideband.band1.bytes=3098
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2893ba9284af4
Request
9b9d36ac…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format