GitHubivarvong
ply
main
1// Fuzz the pack we PRODUCE. For random (repo, path, edit) shapes, build the thin push pack exactly as editFile does
2// and let git judge it: index-pack --fix-thin --strict must accept it, and the resulting objects must be byte-identical
3// to those from a full-object (non-delta) pack of the same edit and to git's own objects after applying the edit with
4// git itself. Targets delta-op boundaries (64 KiB copies, 127-byte inserts), deep paths, odd names, tree sizes.
5// node test/fuzz/packgen.mjs [--n 200] [--seed 1]
6import { join } from 'node:path';
7import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
8import { execFileSync } from 'node:child_process';
9import { git, gitBuf, tmpRoot, fsckProblems } from '../server.js';
10import { Rng } from './prng.js';
11import { parseTree, treeReplaceOid, hashObject, encodeCommit, TYPE } from '../../src/objects.js';
12import { spliceDelta, buildPack, makeDelta } from '../../src/pack.js';
13import { fromHex } from '../../src/sha1.js';
14import { concat, te } from '../../src/bytes.js';
15import { unwrap } from '../unwrap.js';
17const args = Object.fromEntries(process.argv.slice(2).map((a, i, arr) => a.startsWith('--') ? [a.slice(2), arr[i + 1] ?? true] : []).filter(x => x.length));
18const N = parseInt(args.n ?? '200', 10), SEED = parseInt(args.seed ?? String(Date.now() % 1e9), 10);
19// The run seed, before any case: a clock-seeded run is replayable only if it says which seed the clock gave (#308).
20console.log(`packgen: seed ${SEED}, n ${N} replay: node --import ./test/ts-resolve.mjs test/fuzz/packgen.mjs --n ${N} --seed ${SEED}`);
21const { root, cleanup } = tmpRoot();
22const fails = [], cov = new Map();
23let skipped = 0;
24const bump = k => cov.set(k, (cov.get(k) ?? 0) + 1);
25try {
26 for (let i = 0; i < N; i++) {
27 const seed = (SEED * 7919 + i * 104729) >>> 0, rng = new Rng(seed);
28 const work = join(root, `w${i}`); mkdirSync(work); git(work, ['init', '-q', '-b', 'main']);
29 const env = { ...process.env, GIT_AUTHOR_NAME: 'p', GIT_AUTHOR_EMAIL: 'p@x', GIT_COMMITTER_NAME: 'p', GIT_COMMITTER_EMAIL: 'p@x', GIT_AUTHOR_DATE: '1700000000 +0000', GIT_COMMITTER_DATE: '1700000000 +0000', GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'commit.gpgsign', GIT_CONFIG_VALUE_0: 'false' };
30 // shape
31 const depth = rng.weighted([[1, 3], [2, 3], [3, 2], [6, 1], [12, 1]]);
32 const nameOf = () => rng.pick(['a', 'b-c', 'd.e', 'f g', '-h', '.i', 'ü', '日本', 'x'.repeat(rng.int(1, 60)), 'z' + rng.int(0, 99)]);
33 const comps = Array.from({ length: depth }, nameOf); const path = comps.join('/');
34 const blobLen = rng.weighted([[0, 1], [1, 1], [rng.int(2, 200), 4], [rng.int(60000, 70000), 2], [rng.int(130000, 140000), 2], [rng.int(300, 5000), 3]]);
35 const base = rng.bool(0.5) ? Buffer.alloc(blobLen, rng.int(0, 255)) : Buffer.from(Array.from({ length: blobLen }, () => rng.int(0, 255)));
36 const off = rng.pick([0, blobLen, blobLen >> 1, Math.min(blobLen, 65535), Math.min(blobLen, 65536), Math.min(blobLen, 65537), Math.min(blobLen, 131072), rng.int(0, blobLen)]);
37 const oldLen = Math.min(blobLen - off, rng.pick([0, 1, 7, 127, 128, 255, 256, rng.int(0, 300)]));
38 const insert = Buffer.from(Array.from({ length: rng.pick([0, 1, 126, 127, 128, 254, 255, 256, rng.int(0, 2000)]) }, () => rng.int(0, 255)));
39 const target = Buffer.concat([base.subarray(0, off), insert, base.subarray(off + oldLen)]);
40 // noop edits never reach the pack builder; the case still says so, under its seed (#308)
41 if (target.equals(base)) { skipped++; process.stdout.write(`skip #${String(i).padStart(3)} seed=${seed} noop edit depth=${depth} blob=${blobLen} off=${off} old=${oldLen} ins=${insert.length}\n`); rmSync(work, { recursive: true, force: true }); continue; }
42 // siblings so trees are non-trivial
43 const files = { [path]: base };
44 for (let k = rng.int(0, 30); k > 0; k--) { const d = rng.int(0, depth - 1); files[[...comps.slice(0, d), 's' + rng.int(0, 9999)].join('/')] = Buffer.from('s'); }
45 for (const [p, v] of Object.entries(files)) { mkdirSync(join(work, p, '..'), { recursive: true }); writeFileSync(join(work, p), v); }
46 git(work, ['add', '-A']); git(work, ['commit', '-q', '-m', 'base'], { env });
47 const oldCommit = git(work, ['rev-parse', 'HEAD']).trim();
48 // git's own version of the edit (the reference objects)
49 writeFileSync(join(work, path), target); git(work, ['add', '-A']); git(work, ['commit', '-q', '-m', 'edit'], { env });
50 const refCommit = git(work, ['rev-parse', 'HEAD']).trim(), refTree = git(work, ['rev-parse', 'HEAD^{tree}']).trim(), refBlob = git(work, ['rev-parse', `HEAD:${path}`]).trim();
51 git(work, ['reset', '-q', '--hard', oldCommit]);
52 // our objects: walk trees from the old commit (what editFile holds), splice, build thin pack
53 const levels = []; let treeOid = git(work, ['rev-parse', `${oldCommit}^{tree}`]).trim();
54 for (const name of comps) { const data = new Uint8Array(gitBuf(work, ['cat-file', 'tree', treeOid])); const entry = unwrap(parseTree(data)).find(e => e.name === name); levels.push({ treeOid, data, entry }); treeOid = entry.oid; }
55 const oldBlob = treeOid;
56 const newBlob = hashObject('blob', new Uint8Array(target));
57 const blobDelta = spliceDelta({ baseLen: base.length, off, oldLen, insert: new Uint8Array(insert) }).delta;
58 const treeObjs = []; let child = newBlob;
59 for (let l = levels.length - 1; l >= 0; l--) { const { treeOid: oldT, data, entry } = levels[l]; const nd = treeReplaceOid(data, entry, child); const nid = hashObject('tree', nd); treeObjs.unshift({ data: spliceDelta({ baseLen: data.length, off: entry.oidOffset, oldLen: 20, insert: fromHex(child) }).delta, full: nd, baseOid: oldT, newOid: nid }); child = nid; }
60 const commitData = unwrap(encodeCommit({ tree: child, parents: [oldCommit], author: 'p <p@x> 1700000000 +0000', committer: 'p <p@x> 1700000000 +0000', message: 'edit\n' }));
61 const newCommit = hashObject('commit', commitData);
62 const problems = [];
63 if (newBlob !== refBlob) problems.push(`blob oid ${newBlob.slice(0, 8)} != git ${refBlob.slice(0, 8)}`);
64 if (child !== refTree) problems.push(`tree oid ${child.slice(0, 8)} != git ${refTree.slice(0, 8)}`);
65 if (newCommit !== refCommit) problems.push(`commit oid != git (${newCommit.slice(0, 8)} vs ${refCommit.slice(0, 8)})`);
66 const thinObjs = [{ type: TYPE.COMMIT, data: commitData }, ...treeObjs.map(t => ({ type: TYPE.TREE, data: t.data, base: { tag: 'ref', oid: t.baseOid } })), blobDelta.length >= 4 ? { type: TYPE.BLOB, data: blobDelta, base: { tag: 'ref', oid: oldBlob } } : { type: TYPE.BLOB, data: new Uint8Array(target) }];
67 const fullObjs = [{ type: TYPE.COMMIT, data: commitData }, ...treeObjs.map(t => ({ type: TYPE.TREE, data: t.full })), { type: TYPE.BLOB, data: new Uint8Array(target) }];
68 const { pack: thin } = await buildPack(thinObjs), { pack: full } = await buildPack(fullObjs);
69 // git judges: both packs must index strictly in a repo that has only the OLD objects
70 const judge = join(root, `j${i}`); execFileSync('git', ['clone', '-q', '--bare', work, judge]); git(judge, ['update-ref', 'refs/heads/main', oldCommit]); git(judge, ['reflog', 'expire', '--expire=now', '--all']); git(judge, ['gc', '-q', '--prune=now']);
71 for (const [label, pack] of [['thin', thin], ['full', full]]) {
72 try { execFileSync('git', ['-C', judge, 'index-pack', '--fix-thin', '--strict', '--stdin', '--keep'], { input: pack, stdio: 'pipe' }); } // no -o: lands in objects/pack so git can see the objects
73 catch (e) { problems.push(`${label} pack rejected: ${(e.stderr || '').toString().trim().split('\n').pop().slice(0, 100)}`); }
74 }
75 if (!problems.length) {
76 if (!Buffer.from(gitBuf(judge, ['cat-file', 'blob', newBlob])).equals(target)) problems.push('blob content via git != target');
77 const fsck = fsckProblems(judge, { args: [newCommit] }); if (fsck.length) problems.push(`fsck ${fsck[0]}`);
78 // and a real push of the thin pack lands the ref
79 const out = execFileSync('git', ['-C', judge, 'receive-pack', '.'], { input: concat([pktFor(`${oldCommit} ${newCommit} refs/heads/main\0report-status\n`), te.encode('0000'), thin]), stdio: 'pipe' }).toString();
80 if (!/unpack ok/.test(out) || !/ok refs\/heads\/main/.test(out)) problems.push(`receive-pack: ${out.replace(/\n/g, '|').slice(0, 100)}`);
81 if (git(judge, ['rev-parse', 'main']).trim() !== newCommit) problems.push('ref not advanced');
82 }
83 for (const b of [`depth.${depth}`, `blob.${blobLen === 0 ? 0 : 2 ** Math.floor(Math.log2(blobLen))}`, `off.${off === 0 ? 'start' : off === blobLen ? 'end' : off % 65536 === 0 ? '64k-aligned' : off % 65536 === 65535 ? '64k-1' : 'mid'}`, `oldLen.${oldLen}`, `insert.${insert.length}`, `copyops.${Math.ceil(off / 65536) + Math.ceil((blobLen - off - oldLen) / 65536)}`, `thin.${thin.length < full.length ? 'smaller' : 'not-smaller'}`]) bump(b);
84 process.stdout.write(`${problems.length ? 'FAIL' : 'ok '} #${String(i).padStart(3)} seed=${seed} depth=${depth} blob=${blobLen} off=${off} old=${oldLen} ins=${insert.length} thin=${thin.length}B full=${full.length}B\n`);
85 if (problems.length) { fails.push(seed); console.log(' ' + problems.join('\n ')); }
86 rmSync(work, { recursive: true, force: true }); rmSync(judge, { recursive: true, force: true });
87 }
88} finally { cleanup(); }
89function pktFor(s) { const b = te.encode(s); return concat([te.encode((b.length + 4).toString(16).padStart(4, '0')), b]); }
90console.log(`\n${N - skipped} cases, ${skipped} skipped (noop edit), ${fails.length} failures (seed ${SEED})\n` + [...cov.entries()].sort((a, b) => a[1] - b[1] || a[0].localeCompare(b[0])).map(([k, v]) => `${String(v).padStart(4)} ${k}`).join('\n'));
91process.exit(fails.length ? 1 : 0);
Diagnostics7 git calls · 102 ms
CallStatusSizeTime0102 ms
discover200191 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=11990 git.response.cache=UPDATING git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetchtree:0200853 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=87 git.response.bytes=853 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=fb781d4e4fd58ededa1f1078d937de6f4b1316a2 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=853 git.pkt.payload.max=826 git.sideband.band1.bytes=826 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=826 git.pack.trailer=c89bc485f6ad08d875fc515f3fdd69897096847bOpen pack map
fetch2003.8 KiB87 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3909 git.response.cache=MISS git.response.duration_ms=87 git.fetch.wants=255af754fe361e847efd5eea58412439d9cd8ba5 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3909 git.pkt.payload.max=3882 git.sideband.band1.bytes=3882 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2c97acfad4b67
- Request
- 3e256493…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format