GitHubivarvong

ply

main
ply6 git calls78 ms27 KiB4 cachedprotocol v2
142 lines11 KiBRaw
1// One server mode of the exhaustive fetch matrix in test/protocol.test.js, run in a worker thread so the three modes
2// run side by side: each cell blocks on synchronous git (the pack-objects oracle) and the git server lives in the same
3// thread, so one thread can only take the modes one after another. Each worker has its own root, server and hook log.
4// It posts its stats and per-cell object counts; the test sums them and makes the cross-mode assertions.
5import { parentPort, workerData } from 'node:worker_threads';
6import assert from 'node:assert/strict';
7import { join } from 'node:path';
8import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
9import { fileURLToPath } from 'node:url';
10import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer } from './server.js';
11import { genDag, readModel, expectedObjects, requiredAndHad, treeClosure } from './fuzz/dag.js';
12import { packObjectsOracle, reconstructFor } from './fuzz/packobjects.js';
13import { Rng } from './fuzz/prng.js';
14import { GitHttp } from '../src/client.js';
15import { Tracer } from '../src/otel.js';
16import { TYPE_NAME } from '../src/objects.js';
17import { unwrap, unwrapAsync } from './unwrap.js';
19let root, srv, hookLog;
21// The cell plan, without a repository: the four enumerated dimensions by key, and the flags that cycle with the cell
22// number. test/protocol.test.js holds this plan to a stated rule (no flag is a function of an enumerated dimension)
23// before the matrix runs it, so the plan the guard judges is the plan the cells follow.
24export const WANT_KEYS = ['tip', 'old', 'tag', 'tree', 'blob', 'tip+old', 'tip+tree', 'tag+blob'];
25export const HAVE_KEYS = ['none', 'ancestor', 'old', 'branch', 'tip', 'tree', 'blob', 'tag', 'unknown', 'tip+old', 'old+tip'];
26export const DEEPEN_KEYS = ['none', 'd1', 'd2', 'd99', 'since', 'not'];
27export const FILTER = [undefined, 'blob:none', 'blob:limit=100', 'blob:limit=1', 'tree:0', 'tree:1', 'tree:2', 'combine:blob:none+tree:1', 'combine:blob:limit=100+tree:2'];
28// Periods 13, 17 and 5 are coprime to each other, to every stride of the product (1, 9, 54, 594) and to the 7 and 11
29// of the client-shallow cycle, so no flag is a function of a dimension or of another flag. (`i % 3` and `i % 2` under
30// the 9-value filter loop asked for thin-pack with three filters only and gave each (deepen, filter) pair one ofs value.)
31export const flagsOf = i => ({ thinPack: (i % 13) < 6, ofsDelta: (i % 17) < 8, done: (i % 5) !== 0 });
32export function* plan() {
33 let i = 0;
34 for (const wk of WANT_KEYS) for (const hk of HAVE_KEYS) for (const dk of DEEPEN_KEYS) for (const filter of FILTER) { i++; yield { i, wk, hk, dk, filter, ...flagsOf(i) }; }
37function makeServer(mode) {
38 const name = `m-${mode}`;
39 const bare = makeBareRepo(root, name);
40 const work = join(root, `w-${mode}`); mkdirSync(work);
41 const rng = new Rng(4242);
42 const dag = genDag(rng, work, { commits: 16, branches: 3 });
43 git(work, ['push', '-q', '--all', bare]); git(work, ['push', '-q', '--tags', bare]);
44 git(bare, ['config', 'pack.depth', '10']);
45 if (mode !== 'loose') git(bare, ['gc', '-q']);
46 git(bare, ['config', 'pack.useBitmaps', String(mode === 'bitmap')]);
47 return { name, bare, dag, model: readModel(bare), bitmap: mode === 'bitmap', mode };
50export async function runMode(mode) {
51 const S = makeServer(mode);
52 const stats = { cells: 0, modelExact: 0, oracleExact: 0, invocationExact: 0, degenerate: 0, crash: 0 };
53 const norm = t => { const lines = t.trim().split('\n'); const i = lines.indexOf('--not'); const uniq = a => [...new Set(a)].sort(); return JSON.stringify([uniq(lines.slice(0, i).filter(l => l.startsWith('--shallow'))), uniq(lines.slice(0, i).filter(l => !l.startsWith('--shallow'))), uniq(lines.slice(i + 1).filter(Boolean))]); };
54 const seen = new Map();
55 const { model, dag, bare } = S;
56 const C = i => dag.commitOids[i];
57 const tip = dag.tip, old = C(2), mid = C(8), branch = dag.branchHeads.find(h => h !== tip) ?? C(4);
58 const tag = model.refs.get('refs/tags/v1');
59 const tipTree = model.commits.get(tip).tree;
60 const subtree = model.trees.get(tipTree).find(e => e.type === 'tree').oid;
61 const blob = [...treeClosure(model, [tipTree]).keys()].find(o => model.type.get(o) === 'blob');
62 const oldBlob = [...treeClosure(model, [model.commits.get(old).tree]).keys()].find(o => model.type.get(o) === 'blob');
63 const WANTS = { tip: [tip], old: [old], tag: [tag], tree: [subtree], blob: [blob], 'tip+old': [tip, old], 'tip+tree': [tip, subtree], 'tag+blob': [tag, blob] };
64 const HAVES = { none: [], ancestor: [mid], old: [old], branch: [branch], tip: [tip], tree: [subtree], blob: [oldBlob], tag: [tag], unknown: ['e'.repeat(40)], 'tip+old': [tip, old], 'old+tip': [old, tip] };
65 const DEEPEN = { none: {}, d1: { deepen: 1 }, d2: { deepen: 2 }, d99: { deepen: 99 }, since: { deepenSince: model.commits.get(old).date - 500 }, not: { deepenNot: ['refs/tags/v1'] } };
66 const client = unwrap(GitHttp.create(`${srv.url}/${S.name}.git`, { tracer: new Tracer() }));
67 const external = oid => (model.type.has(oid) ? { type: { blob: 3, tree: 2, commit: 1, tag: 4 }[model.type.get(oid)], data: new Uint8Array(gitBuf(bare, ['cat-file', model.type.get(oid), oid])) } : undefined);
68 assert.deepEqual([Object.keys(WANTS), Object.keys(HAVES), Object.keys(DEEPEN)], [WANT_KEYS, HAVE_KEYS, DEEPEN_KEYS], 'the plan names every dimension value');
69 for (const { i, wk, hk, dk, filter, thinPack, ofsDelta, done } of plan()) {
70 const wants = WANTS[wk], haves = HAVES[hk], d = DEEPEN[dk];
71 // client shallow state cycles: none / the have is shallow / the tip is shallow (only meaningful with deepen)
72 const shallow = (i % 7) === 0 && haves.length && model.commits.has(haves[0]) ? [haves[0]] : (i % 11) === 0 && dk !== 'none' ? [tip] : [];
73 const req = { wants, haves, ...d, filter, thinPack, ofsDelta, done, shallows: shallow };
74 const label = `${S.mode} want=${wk} have=${hk} ${dk} ${filter ?? '-'} ${thinPack ? 'thin' : ''}${ofsDelta ? ' ofs' : ' ref'}${done ? '' : ' nego'}${shallow.length ? ' sh' : ''}`;
75 stats.cells++;
76 const wantCommits = wants.map(w => model.tags.get(w) ?? w).filter(w => model.commits.has(w));
77 const exp = expectedObjects(model, { ...req, shallow, bitmap: S.bitmap });
78 const bound = requiredAndHad(model, { ...req, shallow, bitmap: S.bitmap });
79 const degenerate = (d.deepenSince || d.deepenNot) && (wantCommits.length === 0 || wantCommits.some(w => (d.deepenSince && model.commits.get(w).date < d.deepenSince) || (d.deepenNot && !exp.inDepth.has(w))));
80 let r;
81 try { r = done ? await unwrapAsync(client.fetchPack(req, { external })) : await unwrapAsync(client.negotiate(req, { external })); }
82 catch (e) {
83 // The server answers no request of this plan with ERR: it serves the cell or, for a degenerate deepen, dies
84 // without a flush (remoteRefused=0 in every recorded run of the matrix, and in a probe of all 14,256 requests).
85 // A remote_error is therefore ply sending what the server refuses, and it fails its cell like any other error.
86 // It was once counted and skipped in any cell, so a ply refused in every cell passed with nothing compared.
87 if (e.code === 'protocol' && /without flush/.test(e.message) && degenerate) { stats.crash++; continue; }
88 assert.fail(`${label}: ${e.code} ${e.message}`);
89 }
90 if (!r.parsed) assert.fail(`${label}: no pack (acks ${r.acks})`);
91 if (degenerate) { stats.degenerate++; continue; }
92 const got = new Set(r.objects.keys());
93 // bound
94 for (const o of bound.required) assert.ok(got.has(o), `${label}: required ${model.type.get(o)} ${o.slice(0, 8)} missing`);
95 for (const o of got) assert.ok(bound.required.has(o) || bound.had.has(o), `${label}: extra ${model.type.get(o)} ${o.slice(0, 8)} not had`);
96 // model
97 const modelExact = [...exp.objects].every(o => got.has(o)) && [...got].every(o => exp.objects.has(o));
98 if (modelExact) stats.modelExact++;
99 // oracle
100 const shallowSet = new Set([...shallow.filter(c => model.commits.has(c)), ...bound.boundary]);
101 const o = packObjectsOracle(bare, model, req, { shallowSet, unshallowed: bound.unshallow, inDepth: bound.inDepth, external });
102 const oMissing = [...o.objects.keys()].filter(x => !got.has(x)), oExtra = [...got].filter(x => !o.objects.has(x));
103 assert.equal(oMissing.length + oExtra.length, 0, `${label}: pack-objects oracle differs (oracle-only ${oMissing.length}, fetch-only ${oExtra.length})`);
104 stats.oracleExact++;
105 // the invocation upload-pack actually made vs our reconstruction (flags exact; lists compared as sets)
106 const realArgs = readFileSync(hookLog + '.args', 'utf8').trim().replace(/^git --shallow-file \S* pack-objects /, '').replace(/^git pack-objects /, '');
107 const mine = reconstructFor(model, req, { shallowSet, unshallowed: bound.unshallow });
108 assert.equal(realArgs, mine.args.join(' '), `${label}: pack-objects argv`);
109 assert.equal(norm(readFileSync(hookLog + '.stdin', 'utf8')), norm(mine.stdin), `${label}: pack-objects stdin`);
110 stats.invocationExact++;
111 // object integrity
112 for (const [oid, obj] of r.objects) { assert.equal(TYPE_NAME[obj.type], model.type.get(oid), label); assert.equal(obj.data.length, model.size.get(oid), label); }
113 // shallow lines
114 if (dk !== 'none') { assert.deepEqual([...r.shallow].sort(), exp.shallow, `${label}: shallow lines`); assert.deepEqual([...r.unshallow].sort(), exp.unshallow, `${label}: unshallow lines`); }
115 else assert.equal(r.shallow.length + r.unshallow.length, 0, label);
116 // pack properties
117 if (!ofsDelta) assert.equal(r.parsed.stats.ofsDelta, 0, label);
118 if (!thinPack) assert.equal(r.resolved.stats.externalBases, 0, label);
119 // a REF base outside the pack is one of the haves (the read side's tagged base; a `baseOid` field no longer exists)
120 for (const [, obj] of r.objects) { const ref = obj.packBase.tag === 'ref' ? obj.packBase.oid : null; if (ref !== null && !r.objects.has(ref)) assert.ok(bound.had.has(ref), `${label}: thin base not derivable`); }
121 assert.ok(r.resolved.stats.maxChain <= 10, label);
122 seen.set(`${S.mode}|${wk}|${hk}|${dk}|${filter ?? '-'}`, got.size);
123 }
124 return { stats, seen: [...seen] };
127if (parentPort) {
128 const { mode } = workerData;
129 let cleanup;
130 try {
131 ({ root, cleanup } = tmpRoot());
132 const hookCfg = join(root, 'gitconfig'); hookLog = join(root, 'hooklog');
133 writeFileSync(hookCfg, `[uploadpack]\n\tpackObjectsHook = ${fileURLToPath(new URL('./fuzz/pack-objects-hook.sh', import.meta.url))}\n`);
134 srv = await startGitServer(root, { GIT_CONFIG_GLOBAL: hookCfg, PLY_HOOK_LOG: hookLog });
135 parentPort.postMessage({ ok: true, ...(await runMode(mode)) });
136 } catch (e) {
137 parentPort.postMessage({ ok: false, error: `${mode}: ${e.stack ?? e}` });
138 } finally {
139 await srv?.close();
140 cleanup?.();
141 }
Diagnostics6 git calls · 78 ms
CallStatusSizeTime078 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=68
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2004.2 KiB68 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=4319
git.response.cache=MISS
git.response.duration_ms=68
git.fetch.wants=d312f9d38be63053285daf7ad593bd33d1cda652
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4319
git.pkt.payload.max=4292
git.sideband.band1.bytes=4292
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0fb60ab15332c
Request
22ab7fc6…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format