main
1// Pack-reading compliance: git pack-objects over random DAG repos × its option space must parse and resolve to
2// `git cat-file` truth, with offsets and delta-chain depths matching `git verify-pack -v`.
3// node test/fuzz/packs.mjs [--n 60] [--seed 1]
4import { join } from 'node:path';
5import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
6import { git, gitBuf, tmpRoot } from '../server.js';
7import { genDag, readModel } from './dag.js';
8import { Rng } from './prng.js';
9import { parsePack, resolvePack } from '../../src/pack.js';
10import { TYPE_NAME } from '../../src/objects.js';
11import { unwrap } from '../unwrap.js';
13const args = Object.fromEntries(process.argv.slice(2).map((a, i, arr) => a.startsWith('--') ? [a.slice(2), arr[i + 1] ?? true] : []).filter(x => x.length));
14const N = parseInt(args.n ?? '60', 10), SEED = parseInt(args.seed ?? String(Date.now() % 1e9), 10);
15const { root, cleanup } = tmpRoot();
16const cov = new Map(), fails = [];
17let ran = 0, thinJudged = 0, fullJudged = 0, basesJudged = 0;
18console.log(`packs: seed ${SEED}, n ${N} replay: node --import ./test/ts-resolve.mjs test/fuzz/packs.mjs --n ${N} --seed ${SEED}`);
19try {
20 for (let i = 0; i < N; i++) {
21 const seed = (SEED * 7919 + i * 104729) >>> 0, rng = new Rng(seed);
22 const work = join(root, `w${i}`); mkdirSync(work);
23 genDag(rng, work, { commits: rng.int(3, 30), branches: rng.int(1, 4) });
24 // extra large/binary/repetitive blobs so deltas and stored blocks appear
25 if (rng.bool(0.5)) { writeFileSync(join(work, 'big'), Buffer.alloc(rng.int(1, 300) * 1024, rng.int(0, 255))); git(work, ['add', 'big']); git(work, ['-c', 'user.name=x', '-c', 'user.email=x@x', 'commit', '-q', '-m', 'big']); }
26 const model = readModel(work);
27 const truth = new Map(git(work, ['cat-file', '--batch-check', '--batch-all-objects']).trim().split('\n').map(l => { const [oid, type, size] = l.split(' '); return [oid, { type, size: +size }]; }));
28 const opts = {
29 window: rng.pick([0, 1, 10, 50, 250]), depth: rng.pick([0, 1, 5, 50, 250]), compression: rng.pick([-1, 0, 1, 6, 9]),
30 reuseDelta: rng.bool(0.7), reuseObject: rng.bool(0.7), sparse: rng.bool(0.5), ofs: rng.bool(0.7), thin: rng.bool(0.3), bigFile: rng.pick([0, 1024, 1 << 20]),
31 };
32 const haves = opts.thin ? [...model.commits.keys()].filter(() => rng.bool(0.2)) : [];
33 const wants = [...model.refs.values()];
34 const stdin = [...wants, ...(haves.length ? ['--not', ...haves] : []), ''].join('\n') + '\n';
35 const argv = ['-c', `pack.compression=${opts.compression}`, ...(opts.bigFile ? ['-c', `core.bigFileThreshold=${opts.bigFile}`] : []), 'pack-objects', '--revs', '--stdout', '-q', `--window=${opts.window}`, `--depth=${opts.depth}`, ...(opts.ofs ? ['--delta-base-offset'] : []), ...(opts.thin ? ['--thin'] : []), ...(opts.reuseDelta ? [] : ['--no-reuse-delta']), ...(opts.reuseObject ? [] : ['--no-reuse-object']), opts.sparse ? '--sparse' : '--no-sparse'];
36 if (rng.bool(0.4)) git(work, ['gc', '-q']); // packed on disk → delta/object reuse paths
37 const pack = new Uint8Array(gitBuf(work, argv, stdin));
38 const problems = [];
39 let parsed, r;
40 try {
41 parsed = unwrap(parsePack(pack));
42 const asked = new Map(); // every oid ply looked up outside the pack, with how often
43 const external = oid => (asked.set(oid, (asked.get(oid) ?? 0) + 1), truth.has(oid) ? { type: { blob: 3, tree: 2, commit: 1, tag: 4 }[truth.get(oid).type], data: new Uint8Array(gitBuf(work, ['cat-file', truth.get(oid).type, oid])) } : undefined);
44 r = unwrap(resolvePack(parsed, external));
45 // the pack's own entry count (pack-format: a big-endian uint32 at bytes 8..11), read here, not by ply
46 const header = Buffer.from(pack.buffer, pack.byteOffset, 12).readUInt32BE(8);
47 if (r.objects.size !== header) problems.push(`resolved ${r.objects.size} objects, the pack header says ${header}`);
48 for (const [oid, o] of r.objects) {
49 const t = truth.get(oid);
50 if (!t) { problems.push(`unknown object ${oid}`); break; }
51 if (TYPE_NAME[o.type] !== t.type || o.data.length !== t.size) { problems.push(`type/size ${oid}`); break; }
52 if (rng.bool(0.3) && !Buffer.from(o.data).equals(gitBuf(work, ['cat-file', t.type, oid]))) { problems.push(`content ${oid}`); break; }
53 }
54 // expected set: rev-list of the same revs (non-thin and thin alike: --not excludes; pack-objects' edge heuristics may add)
55 const exp = new Set(git(work, ['rev-list', '--objects', '--no-object-names', ...wants, ...haves.map(h => `^${h}`)]).trim().split('\n').filter(Boolean));
56 for (const o of exp) if (!r.objects.has(o)) { problems.push(`missing required ${o}`); break; }
57 if (!opts.ofs && parsed.stats.ofsDelta) problems.push('ofs delta without --delta-base-offset');
58 if (!opts.thin && r.stats.externalBases) problems.push('external base in non-thin pack');
59 if (opts.depth && r.stats.maxChain > opts.depth) problems.push(`chain ${r.stats.maxChain} > depth ${opts.depth}`);
60 // git's own listing of this pack, not ply's: offsets, chain depths and the exact object set. A thin pack is first
61 // completed by `index-pack --fix-thin`, which appends each base the pack lacks after the pack's own entries and
62 // moves none of them, so an entry at or past the original trailer (pack.length - 20) is an appended base and
63 // every entry before it is an object of the pack as sent. A non-thin pack has nothing appended.
64 const pdir = join(root, `vp${i}`); mkdirSync(pdir);
65 const listed = join(pdir, opts.thin ? 'fixed.pack' : 'p.pack');
66 if (opts.thin) gitBuf(work, ['index-pack', '--fix-thin', '--stdin', listed], pack);
67 else { writeFileSync(listed, pack); git(work, ['index-pack', listed]); }
68 const own = new Map(), appended = new Set();
69 for (const l of git(work, ['verify-pack', '-v', listed]).split('\n').filter(l => /^[0-9a-f]{40} (commit|tree|blob|tag)/.test(l))) {
70 const [oid, , , , offset, chain] = l.trim().split(/\s+/);
71 if (+offset < pack.length - 20) own.set(oid, { offset: +offset, chain: chain ? +chain : 0 }); else appended.add(oid);
72 }
73 for (const [oid, g] of own) {
74 const o = r.objects.get(oid);
75 if (!o || o.offset !== g.offset || o.chain !== g.chain) { problems.push(`verify-pack offset/chain ${oid.slice(0, 8)}: git ${g.offset}/${g.chain}, ply ${o ? `${o.offset}/${o.chain}` : 'no such object'}`); break; }
76 }
77 for (const oid of r.objects.keys()) if (!own.has(oid)) { problems.push(`object ${oid} is not an entry of the pack in git's listing${appended.has(oid) ? ' (it is a base git appended)' : ''}`); break; }
78 // the bases: ply must have asked for exactly the ones git had to append
79 for (const oid of appended) if (!asked.has(oid)) { problems.push(`git appended base ${oid}, ply never asked for it`); break; }
80 for (const oid of asked.keys()) if (!appended.has(oid)) { problems.push(`ply asked for base ${oid}, git did not need it`); break; }
81 if (!opts.thin && appended.size) problems.push('git appended a base to a non-thin pack');
82 if (opts.thin) thinJudged++; else fullJudged++;
83 basesJudged += appended.size;
84 } catch (e) { problems.push(`ERROR ${e.code ?? ''} ${e.message.slice(0, 120)}`); }
85 const buckets = [`window.${opts.window}`, `depth.${opts.depth}`, `comp.${opts.compression}`, `ofs.${opts.ofs}`, `thin.${opts.thin}`, `sparse.${opts.sparse}`, `reuseDelta.${opts.reuseDelta}`, `reuseObject.${opts.reuseObject}`, `bigFile.${opts.bigFile}`, `objects.${parsed ? 2 ** Math.floor(Math.log2(parsed.stats.count)) : 'err'}`, `chain.${r ? Math.min(r.stats.maxChain, 9) : 'err'}`, `stored.${parsed?.stats.inflate.stored > 0}`, `ref_delta.${parsed?.stats.refDelta > 0}`, `ofs_delta.${parsed?.stats.ofsDelta > 0}`];
86 for (const b of buckets) cov.set(b, (cov.get(b) ?? 0) + 1);
87 process.stdout.write(`${problems.length ? 'FAIL' : 'ok '} #${String(i).padStart(2)} seed=${seed} objs=${parsed?.stats.count ?? '-'} chain=${r?.stats.maxChain ?? '-'} ${JSON.stringify(opts)}\n`);
88 ran++;
89 if (problems.length) { fails.push(seed); console.log(' ' + problems.join('\n ')); }
90 rmSync(work, { recursive: true, force: true });
91 }
92} finally { cleanup(); }
93console.log(`\n${ran} packs (seed ${SEED}), ${fails.length} failures${fails.length ? ` (case seeds ${fails.join(', ')})` : ''}; object set, offsets and chains equal to git verify-pack: ${fullJudged} whole, ${thinJudged} thin after index-pack --fix-thin (${basesJudged} appended bases). Coverage:\n` + [...cov.entries()].sort((a, b) => a[1] - b[1] || a[0].localeCompare(b[0])).map(([k, v]) => `${String(v).padStart(4)} ${k}`).join('\n'));
94process.exit(fails.length ? 1 : 0);
Diagnostics4 git calls · 221 ms
CallStatusSizeTime0221 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2003.5 KiB216 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=216 git.response.bytes=3554 git.response.cache=MISS git.response.duration_ms=216 git.fetch.wants=0d8136b9c90bbcb160b1709232f83b02b4cb6fb6 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3554 git.pkt.payload.max=3533 git.sideband.band1.bytes=3532 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2fcd47ed6cf81
- Request
- 323bc625…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format