GitHubivarvong

ply

main
ply7 git calls78 ms27 KiB5 cachedprotocol v2
94 lines8.5 KiBRaw
1// Pack-reading compliance: git pack-objects over random DAG repos × its option space must parse and resolve to
2// `git cat-file` truth, with offsets and delta-chain depths matching `git verify-pack -v`.
3// node test/fuzz/packs.mjs [--n 60] [--seed 1]
4import { join } from 'node:path';
5import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
6import { git, gitBuf, tmpRoot } from '../server.js';
7import { genDag, readModel } from './dag.js';
8import { Rng } from './prng.js';
9import { parsePack, resolvePack } from '../../src/pack.js';
10import { TYPE_NAME } from '../../src/objects.js';
11import { unwrap } from '../unwrap.js';
13const args = Object.fromEntries(process.argv.slice(2).map((a, i, arr) => a.startsWith('--') ? [a.slice(2), arr[i + 1] ?? true] : []).filter(x => x.length));
14const N = parseInt(args.n ?? '60', 10), SEED = parseInt(args.seed ?? String(Date.now() % 1e9), 10);
15const { root, cleanup } = tmpRoot();
16const cov = new Map(), fails = [];
17let ran = 0, thinJudged = 0, fullJudged = 0, basesJudged = 0;
18console.log(`packs: seed ${SEED}, n ${N} replay: node --import ./test/ts-resolve.mjs test/fuzz/packs.mjs --n ${N} --seed ${SEED}`);
19try {
20 for (let i = 0; i < N; i++) {
21 const seed = (SEED * 7919 + i * 104729) >>> 0, rng = new Rng(seed);
22 const work = join(root, `w${i}`); mkdirSync(work);
23 genDag(rng, work, { commits: rng.int(3, 30), branches: rng.int(1, 4) });
24 // extra large/binary/repetitive blobs so deltas and stored blocks appear
25 if (rng.bool(0.5)) { writeFileSync(join(work, 'big'), Buffer.alloc(rng.int(1, 300) * 1024, rng.int(0, 255))); git(work, ['add', 'big']); git(work, ['-c', 'user.name=x', '-c', 'user.email=x@x', 'commit', '-q', '-m', 'big']); }
26 const model = readModel(work);
27 const truth = new Map(git(work, ['cat-file', '--batch-check', '--batch-all-objects']).trim().split('\n').map(l => { const [oid, type, size] = l.split(' '); return [oid, { type, size: +size }]; }));
28 const opts = {
29 window: rng.pick([0, 1, 10, 50, 250]), depth: rng.pick([0, 1, 5, 50, 250]), compression: rng.pick([-1, 0, 1, 6, 9]),
30 reuseDelta: rng.bool(0.7), reuseObject: rng.bool(0.7), sparse: rng.bool(0.5), ofs: rng.bool(0.7), thin: rng.bool(0.3), bigFile: rng.pick([0, 1024, 1 << 20]),
31 };
32 const haves = opts.thin ? [...model.commits.keys()].filter(() => rng.bool(0.2)) : [];
33 const wants = [...model.refs.values()];
34 const stdin = [...wants, ...(haves.length ? ['--not', ...haves] : []), ''].join('\n') + '\n';
35 const argv = ['-c', `pack.compression=${opts.compression}`, ...(opts.bigFile ? ['-c', `core.bigFileThreshold=${opts.bigFile}`] : []), 'pack-objects', '--revs', '--stdout', '-q', `--window=${opts.window}`, `--depth=${opts.depth}`, ...(opts.ofs ? ['--delta-base-offset'] : []), ...(opts.thin ? ['--thin'] : []), ...(opts.reuseDelta ? [] : ['--no-reuse-delta']), ...(opts.reuseObject ? [] : ['--no-reuse-object']), opts.sparse ? '--sparse' : '--no-sparse'];
36 if (rng.bool(0.4)) git(work, ['gc', '-q']); // packed on disk → delta/object reuse paths
37 const pack = new Uint8Array(gitBuf(work, argv, stdin));
38 const problems = [];
39 let parsed, r;
40 try {
41 parsed = unwrap(parsePack(pack));
42 const asked = new Map(); // every oid ply looked up outside the pack, with how often
43 const external = oid => (asked.set(oid, (asked.get(oid) ?? 0) + 1), truth.has(oid) ? { type: { blob: 3, tree: 2, commit: 1, tag: 4 }[truth.get(oid).type], data: new Uint8Array(gitBuf(work, ['cat-file', truth.get(oid).type, oid])) } : undefined);
44 r = unwrap(resolvePack(parsed, external));
45 // the pack's own entry count (pack-format: a big-endian uint32 at bytes 8..11), read here, not by ply
46 const header = Buffer.from(pack.buffer, pack.byteOffset, 12).readUInt32BE(8);
47 if (r.objects.size !== header) problems.push(`resolved ${r.objects.size} objects, the pack header says ${header}`);
48 for (const [oid, o] of r.objects) {
49 const t = truth.get(oid);
50 if (!t) { problems.push(`unknown object ${oid}`); break; }
51 if (TYPE_NAME[o.type] !== t.type || o.data.length !== t.size) { problems.push(`type/size ${oid}`); break; }
52 if (rng.bool(0.3) && !Buffer.from(o.data).equals(gitBuf(work, ['cat-file', t.type, oid]))) { problems.push(`content ${oid}`); break; }
53 }
54 // expected set: rev-list of the same revs (non-thin and thin alike: --not excludes; pack-objects' edge heuristics may add)
55 const exp = new Set(git(work, ['rev-list', '--objects', '--no-object-names', ...wants, ...haves.map(h => `^${h}`)]).trim().split('\n').filter(Boolean));
56 for (const o of exp) if (!r.objects.has(o)) { problems.push(`missing required ${o}`); break; }
57 if (!opts.ofs && parsed.stats.ofsDelta) problems.push('ofs delta without --delta-base-offset');
58 if (!opts.thin && r.stats.externalBases) problems.push('external base in non-thin pack');
59 if (opts.depth && r.stats.maxChain > opts.depth) problems.push(`chain ${r.stats.maxChain} > depth ${opts.depth}`);
60 // git's own listing of this pack, not ply's: offsets, chain depths and the exact object set. A thin pack is first
61 // completed by `index-pack --fix-thin`, which appends each base the pack lacks after the pack's own entries and
62 // moves none of them, so an entry at or past the original trailer (pack.length - 20) is an appended base and
63 // every entry before it is an object of the pack as sent. A non-thin pack has nothing appended.
64 const pdir = join(root, `vp${i}`); mkdirSync(pdir);
65 const listed = join(pdir, opts.thin ? 'fixed.pack' : 'p.pack');
66 if (opts.thin) gitBuf(work, ['index-pack', '--fix-thin', '--stdin', listed], pack);
67 else { writeFileSync(listed, pack); git(work, ['index-pack', listed]); }
68 const own = new Map(), appended = new Set();
69 for (const l of git(work, ['verify-pack', '-v', listed]).split('\n').filter(l => /^[0-9a-f]{40} (commit|tree|blob|tag)/.test(l))) {
70 const [oid, , , , offset, chain] = l.trim().split(/\s+/);
71 if (+offset < pack.length - 20) own.set(oid, { offset: +offset, chain: chain ? +chain : 0 }); else appended.add(oid);
72 }
73 for (const [oid, g] of own) {
74 const o = r.objects.get(oid);
75 if (!o || o.offset !== g.offset || o.chain !== g.chain) { problems.push(`verify-pack offset/chain ${oid.slice(0, 8)}: git ${g.offset}/${g.chain}, ply ${o ? `${o.offset}/${o.chain}` : 'no such object'}`); break; }
76 }
77 for (const oid of r.objects.keys()) if (!own.has(oid)) { problems.push(`object ${oid} is not an entry of the pack in git's listing${appended.has(oid) ? ' (it is a base git appended)' : ''}`); break; }
78 // the bases: ply must have asked for exactly the ones git had to append
79 for (const oid of appended) if (!asked.has(oid)) { problems.push(`git appended base ${oid}, ply never asked for it`); break; }
80 for (const oid of asked.keys()) if (!appended.has(oid)) { problems.push(`ply asked for base ${oid}, git did not need it`); break; }
81 if (!opts.thin && appended.size) problems.push('git appended a base to a non-thin pack');
82 if (opts.thin) thinJudged++; else fullJudged++;
83 basesJudged += appended.size;
84 } catch (e) { problems.push(`ERROR ${e.code ?? ''} ${e.message.slice(0, 120)}`); }
85 const buckets = [`window.${opts.window}`, `depth.${opts.depth}`, `comp.${opts.compression}`, `ofs.${opts.ofs}`, `thin.${opts.thin}`, `sparse.${opts.sparse}`, `reuseDelta.${opts.reuseDelta}`, `reuseObject.${opts.reuseObject}`, `bigFile.${opts.bigFile}`, `objects.${parsed ? 2 ** Math.floor(Math.log2(parsed.stats.count)) : 'err'}`, `chain.${r ? Math.min(r.stats.maxChain, 9) : 'err'}`, `stored.${parsed?.stats.inflate.stored > 0}`, `ref_delta.${parsed?.stats.refDelta > 0}`, `ofs_delta.${parsed?.stats.ofsDelta > 0}`];
86 for (const b of buckets) cov.set(b, (cov.get(b) ?? 0) + 1);
87 process.stdout.write(`${problems.length ? 'FAIL' : 'ok '} #${String(i).padStart(2)} seed=${seed} objs=${parsed?.stats.count ?? '-'} chain=${r?.stats.maxChain ?? '-'} ${JSON.stringify(opts)}\n`);
88 ran++;
89 if (problems.length) { fails.push(seed); console.log(' ' + problems.join('\n ')); }
90 rmSync(work, { recursive: true, force: true });
91 }
92} finally { cleanup(); }
93console.log(`\n${ran} packs (seed ${SEED}), ${fails.length} failures${fails.length ? ` (case seeds ${fails.join(', ')})` : ''}; object set, offsets and chains equal to git verify-pack: ${fullJudged} whole, ${thinJudged} thin after index-pack --fix-thin (${basesJudged} appended bases). Coverage:\n` + [...cov.entries()].sort((a, b) => a[1] - b[1] || a[0].localeCompare(b[0])).map(([k, v]) => `${String(v).padStart(4)} ${k}`).join('\n'));
94process.exit(fails.length ? 1 : 0);
Diagnostics7 git calls · 78 ms
CallStatusSizeTime078 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetchtree:0200853 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=853
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=fb781d4e4fd58ededa1f1078d937de6f4b1316a2
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=853
git.pkt.payload.max=826
git.sideband.band1.bytes=826
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=826
git.pack.trailer=c89bc485f6ad08d875fc515f3fdd69897096847b
Open pack map
fetch2003.5 KiB65 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=65
git.response.bytes=3553
git.response.cache=MISS
git.response.duration_ms=65
git.fetch.wants=0d8136b9c90bbcb160b1709232f83b02b4cb6fb6
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3553
git.pkt.payload.max=3526
git.sideband.band1.bytes=3526
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e9e3f5294ef3c9
Request
72202427…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format