GitHubivarvong
ply
main
1// Issue #46: the blob:none fallback applies the caller's limits to the objects the caller asked for, never to the
2// extras the widening brought. One sibling directory over maxTreeEntries used to fail every read on a blob:none-only
3// host (Cloudflare Artifacts: curl's tests/data, 2,135 entries, under the viewer's 2,000) while a tree:0 server answered.
4//
5// The space, as tagged unions, and every cell of their product (5 × 3 × 4 × 2 = 120), each on a fresh copy of the repo:
6// server: tree0 (git http-backend as is) | blobnone (a proxy that 400s every filter but blob:none, as Artifacts does)
7// wanted: small (a tree under the limit: small/) | big (a tree over the limit: big/) | blob (top.txt; only the root
8// tree is on the way to it)
9// sibling: none | over (huge/, over the limit) | malformed (bad/, entries out of order: git fsck refuses it)
10// | many (huge1..3 over the limit beside ok1, ok2 under it)
11// op: readFile | listTree | editFile | checkoutTree | merge (mergeBranch: the merge-base walk, fetchPackTree0,
12// then the tree merge), each pointed at `wanted` (a file in it, the directory itself, or edits in it)
13// Expected outcome per cell (`expected` below):
14// - a malformed sibling on blobnone: pack_corrupt (#18/#37: a server that sent it is not trusted), except
15// checkoutTree, whose own blob:none pack parses only the trees it walks (on both servers);
16// - else wanted big: limit_exceeded (the wanted tree gets the caller's limits, also when it was skipped from an
17// earlier wide pack and is now fetched on its own);
18// - else listTree/checkoutTree of the file top.txt: not_found;
19// - else ok, and git agrees (cat-file, ls-tree, the edited blob, merge-tree --write-tree).
20// Wherever the sibling is not malformed, tree0 and blobnone give the same answer, oid for oid.
21import { test, beforeAll as before, afterAll as after } from 'vitest';
22import assert from 'node:assert/strict';
23import { join } from 'node:path';
24import { cpSync } from 'node:fs';
25import { createHash } from 'node:crypto';
26import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, fsckProblems } from './server.js';
27import { startProxy } from './proxy.js';
28import { editFile } from '../src/edit.js';
29import { readFile, listTree } from '../src/read.js';
30import { checkoutTree } from '../src/checkout.js';
31import { unwrapAsync } from './unwrap.js';
32import { mergeBranch } from '../src/merge.js';
33import { Tracer, arrayExporter, flattenLine } from '../src/otel.js';
34import { withLimits } from '../src/limits.js';
36const L = 10; // maxTreeEntries; every "over" directory holds L + 1 entries, the issue's core cell (limit = size − 1)
37const LIMITS = withLimits({ maxTreeEntries: L });
38const A = { name: 'w', email: 'w@x' };
39const TIME = 1_700_000_000;
40const ENV = { GIT_AUTHOR_NAME: 's', GIT_AUTHOR_EMAIL: 's@x', GIT_AUTHOR_DATE: `${TIME} +0000`, GIT_COMMITTER_NAME: 's', GIT_COMMITTER_EMAIL: 's@x', GIT_COMMITTER_DATE: `${TIME} +0000` };
41const REJECT = 'Unsupported Git fetch filter; only blob:none is supported.';
42const filterOf = body => /filter ([^\n]+)\n/.exec(Buffer.from(body).toString('latin1'))?.[1] ?? null;
43const artifactsFault = ctx => (ctx.stage === 'fetch' && filterOf(ctx.reqBody) && filterOf(ctx.reqBody) !== 'blob:none' ? { type: 'status', status: 400, contentType: 'text/plain; charset=utf-8', body: REJECT } : { type: 'pass' });
45export const SERVERS = ['tree0', 'blobnone'];
46export const WANTED = ['small', 'big', 'blob'];
47export const SIBLINGS = ['none', 'over', 'malformed', 'many'];
48export const OPS = ['readFile', 'listTree', 'editFile', 'checkoutTree', 'merge'];
50export function expected({ server, op, wanted, sibling }) {
51 if (sibling === 'malformed' && server === 'blobnone' && op !== 'checkoutTree') return 'pack_corrupt';
52 if (wanted === 'big') return 'limit_exceeded';
53 if (wanted === 'blob' && (op === 'listTree' || op === 'checkoutTree')) return 'not_found';
54 return 'ok';
55}
56const overCount = ({ wanted, sibling }) => (wanted === 'big' ? 1 : 0) + ({ none: 0, over: 1, malformed: 0, many: 3 })[sibling];
58// ---- the repository, built with plumbing so both servers' copies (and every run) have the same oids ----
59const hashBlob = (bare, text) => gitBuf(bare, ['hash-object', '-w', '--stdin'], Buffer.from(text)).toString().trim();
60function writeTree(bare, spec) {
61 const lines = Object.entries(spec).map(([name, v]) => {
62 if (typeof v === 'string') return `100644 blob ${hashBlob(bare, v)}\t${name}`;
63 if (v.raw) return `040000 tree ${v.raw}\t${name}`;
64 return `040000 tree ${writeTree(bare, v)}\t${name}`;
65 });
66 return gitBuf(bare, ['mktree'], Buffer.from(lines.join('\n') + '\n')).toString().trim();
67}
68// A tree git fsck refuses (treeNotSorted) but upload-pack serves: written --literally, entries b then a.
69function badTree(bare) {
70 const e = (name, oid) => Buffer.concat([Buffer.from(`100644 ${name}\0`), Buffer.from(oid, 'hex')]);
71 return gitBuf(bare, ['hash-object', '-t', 'tree', '--literally', '-w', '--stdin'], Buffer.concat([e('b', hashBlob(bare, 'b\n')), e('a', hashBlob(bare, 'a\n'))])).toString().trim();
72}
73const commit = (bare, tree, parents, msg) => git(bare, ['commit-tree', tree, ...parents.flatMap(p => ['-p', p]), '-m', msg], { env: { ...process.env, ...ENV } }).trim();
74const dirOf = (tag, n = L + 1) => Object.fromEntries(Array.from({ length: n }, (_, i) => [`f${i}.txt`, `${tag}${i}\nmore\n`]));
76// The target, per `wanted`: the file each op reads or edits, the directory listTree/checkoutTree name, and the two
77// non-overlapping edits the merge's sides make.
78const TARGET = {
79 small: { file: 'small/a.txt', find: 'a1', dir: 'small', feat: s => { s.small['a.txt'] = 'a1 feat\na2\na3\n'; }, main: s => { s.small['b.txt'] = 'b main\n'; } },
80 big: { file: 'big/f0.txt', find: 'big0', dir: 'big', feat: s => { s.big['f0.txt'] = 'big0 feat\nmore\n'; }, main: s => { s.big['f1.txt'] = 'big1 main\nmore\n'; } },
81 blob: { file: 'top.txt', find: 'l1', dir: 'top.txt', feat: s => { s['top.txt'] = 'l1 feat\nl2\nl3\nl4\nl5\n'; }, main: s => { s['top.txt'] = 'l1\nl2\nl3\nl4\nl5 main\n'; } },
82};
83function spec(wanted, sibling, bad) {
84 const s = { 'top.txt': 'l1\nl2\nl3\nl4\nl5\n', small: { 'a.txt': 'a1\na2\na3\n', 'b.txt': 'b\n' } };
85 if (wanted === 'big') s.big = dirOf('big');
86 if (sibling === 'over') s.huge = dirOf('huge');
87 if (sibling === 'malformed') s.bad = { raw: bad };
88 if (sibling === 'many') { for (const k of [1, 2, 3]) s[`huge${k}`] = dirOf(`huge${k}_`); s.ok1 = { 'x.txt': 'x\n' }; s.ok2 = { 'y.txt': 'y\n' }; }
89 return s;
90}
91// base C0; feat = C0 + the feat edit; main = C0 + the main edit (merge merges feat into main; the rest read main)
92function template(root, wanted, sibling) {
93 const name = `tpl-${wanted}-${sibling}`; const bare = makeBareRepo(root, name);
94 const bad = sibling === 'malformed' ? badTree(bare) : null;
95 const c0 = commit(bare, writeTree(bare, spec(wanted, sibling, bad)), [], 'base');
96 const f = spec(wanted, sibling, bad); TARGET[wanted].feat(f); const c1 = commit(bare, writeTree(bare, f), [c0], 'feat');
97 const m = spec(wanted, sibling, bad); TARGET[wanted].main(m); const c2 = commit(bare, writeTree(bare, m), [c0], 'main');
98 git(bare, ['update-ref', 'refs/heads/main', c2]); git(bare, ['update-ref', 'refs/heads/feat', c1]);
99 if (sibling === 'malformed') assert.match(fsckErr(bare), /treeNotSorted|not properly sorted/, 'git refuses the bad tree');
100 return bare;
101}
102const fsckErr = bare => fsckProblems(bare).join('\n');
104// ---- one cell ----
105const sha = b => createHash('sha256').update(b).digest('hex').slice(0, 12);
106const drain = async stream => { const out = []; for await (const c of stream) out.push(Buffer.from(c)); return Buffer.concat(out); };
107async function runOp(op, url, wanted, opts) {
108 const T = TARGET[wanted];
109 if (op === 'readFile') { const r = await unwrapAsync(readFile(url, { path: T.file }, opts)); const b = await drain(r.stream); return { answer: `blob ${r.blob} ${sha(b)}`, check: bare => assert.ok(b.equals(gitBuf(bare, ['show', `main:${T.file}`]))) }; }
110 if (op === 'listTree') {
111 const r = await unwrapAsync(listTree(url, { path: T.dir }, opts));
112 return { answer: `tree ${r.tree} ${sha(JSON.stringify(r.entries))}`, check: bare => assert.deepEqual(r.entries.map(e => `${e.mode} ${e.type} ${e.oid}\t${e.name}`), git(bare, ['ls-tree', `main:${T.dir}`]).trim().split('\n')) };
113 }
114 if (op === 'editFile') {
115 const r = await unwrapAsync(editFile(url, { path: T.file, find: T.find, replace: `${T.find}X`, author: A, message: 'edit', time: TIME }, opts));
116 return { answer: `commit ${r.newCommit}`, check: bare => { assert.equal(git(bare, ['rev-parse', 'main']).trim(), r.newCommit); assert.ok(gitBuf(bare, ['show', `main:${T.file}`]).toString().startsWith(`${T.find}X`)); } };
117 }
118 if (op === 'checkoutTree') {
119 const r = await unwrapAsync(checkoutTree(url, { path: T.dir }, opts));
120 const files = []; for await (const f of r.files) files.push(`${f.path} ${f.oid} ${sha(await drain(f.stream))}`);
121 return { answer: `tree ${r.tree} ${sha(files.join('\n'))}`, check: bare => assert.deepEqual(r.entries.map(e => `${e.mode} blob ${e.oid}\t${e.path}`), git(bare, ['ls-tree', '-r', `main:${T.dir}`]).trim().split('\n')) };
122 }
123 const r = await unwrapAsync(mergeBranch(url, { ref: 'refs/heads/main', from: 'refs/heads/feat', author: A, message: 'merge', time: TIME }, opts));
124 return { answer: `${r.outcome} ${r.oid} ${r.tree}`, check: bare => { assert.equal(r.outcome, 'merge'); assert.equal(r.tree, git(bare, ['merge-tree', '--write-tree', `${r.oid}^1`, `${r.oid}^2`]).trim().split('\n')[0]); assert.equal(git(bare, ['rev-parse', 'main']).trim(), r.oid); } };
125}
127let root, cleanup, srv, proxy, n = 0;
128const templates = new Map();
129before(async () => {
130 ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root); proxy = await startProxy(srv.url, artifactsFault);
131 for (const w of WANTED) for (const s of SIBLINGS) templates.set(`${w}/${s}`, template(root, w, s));
132});
133after(async () => { await proxy.close(); await srv.close(); cleanup(); });
135async function cell(c) {
136 const name = `c${n++}`; const bare = join(root, `${name}.git`);
137 cpSync(templates.get(`${c.wanted}/${c.sibling}`), bare, { recursive: true });
138 const ex = arrayExporter(); const quirks = {};
139 const url = `${c.server === 'tree0' ? srv.url : proxy.url}/${name}.git`;
140 let got, answer = null, err = null;
141 try { const r = await runOp(c.op, url, c.wanted, { tracer: new Tracer({ exporter: ex }), limits: LIMITS, quirks }); answer = r.answer; r.check(bare); got = 'ok'; }
142 catch (e) { if (e?.code === 'ERR_ASSERTION') throw e; err = e; got = e?.code ?? String(e); answer = `${e?.code}: ${String(e?.message).replace(/[0-9a-f]{40}/g, '<oid>')}`; }
143 const fetches = ex.lines.flatMap(flattenLine).filter(s => s.name === 'git.upload-pack fetch');
144 const wide = fetches.filter(s => s.attrs['git.fetch.filter.fallback'] === 'blob:none');
145 const skipped = wide.map(s => s.attrs['git.fetch.wide.skipped'] ?? 0);
146 return { ...c, got, answer, err, skipped, wide, learned: quirks.treeFilter ?? '' };
147}
149test('#46 every cell of server × wanted × sibling × op: the caller\'s limits bind what it asked for, never the extras; tree:0 and blob:none agree', { timeout: 300_000 }, async t => {
150 const rows = [];
151 for (const op of OPS) for (const wanted of WANTED) for (const sibling of SIBLINGS) for (const server of SERVERS) rows.push(await cell({ server, op, wanted, sibling }));
152 const wrong = [];
153 const by = new Map(rows.map(r => [`${r.server}/${r.op}/${r.wanted}/${r.sibling}`, r]));
154 for (const r of rows) {
155 const want = expected(r);
156 if (r.got !== want) wrong.push(`${r.server} ${r.op} ${r.wanted} ${r.sibling}: expected ${want}, got ${r.answer}`);
157 if (r.server === 'blobnone' && r.op !== 'checkoutTree' && r.learned !== 'blob:none') wrong.push(`${r.op} ${r.wanted} ${r.sibling}: blob:none not learned`);
158 if (r.server === 'tree0' && r.wide.length) wrong.push(`tree0 ${r.op} ${r.wanted} ${r.sibling}: a blob:none fallback against a tree:0 server`);
159 // the skipped extras are on the span: exactly the over-limit trees in the first wide pack of a single-tip read, any
160 // in the walk's; none at all where there are none
161 if (r.server === 'blobnone' && r.op !== 'checkoutTree' && r.sibling !== 'malformed') {
162 const k = overCount(r), total = r.skipped.reduce((a, b) => a + b, 0);
163 if (['readFile', 'listTree', 'editFile'].includes(r.op) && r.skipped[0] !== k) wrong.push(`${r.op} ${r.wanted} ${r.sibling}: first wide fetch skipped ${r.skipped[0]}, expected ${k}`);
164 if (k === 0 && total !== 0) wrong.push(`${r.op} ${r.wanted} ${r.sibling}: skipped ${total} with nothing over the limit`);
165 if (k > 0 && !(total > 0 && r.wide.some(s => s.attrs['git.fetch.wide.skipped_reason'] === 'tree entries'))) wrong.push(`${r.op} ${r.wanted} ${r.sibling}: over-limit extras not counted as skipped (tree entries)`);
166 }
167 }
168 // parity: wherever the sibling is well formed, the two servers give the same answer (oids, bytes, error code)
169 for (const op of OPS) for (const wanted of WANTED) for (const sibling of SIBLINGS) {
170 if (sibling === 'malformed') continue;
171 const a = by.get(`tree0/${op}/${wanted}/${sibling}`), b = by.get(`blobnone/${op}/${wanted}/${sibling}`);
172 const key = x => (x.got === 'ok' ? x.answer : x.got);
173 if (key(a) !== key(b)) wrong.push(`parity ${op} ${wanted} ${sibling}: tree0 ${a.answer} | blobnone ${b.answer}`);
174 }
175 console.log(`cells ${rows.length}, as expected ${rows.length - new Set(wrong.filter(w => !w.startsWith('parity')).map(w => w.split(':')[0])).size}`);
176 for (const r of rows) console.log(`CELL ${r.op.padEnd(12)} ${r.wanted.padEnd(5)} ${r.sibling.padEnd(9)} ${r.server.padEnd(8)} expect ${expected(r).padEnd(14)} got ${r.got.padEnd(14)} skipped [${r.skipped}] ${r.got === 'ok' ? '' : r.answer.slice(0, 90)}`);
177 assert.deepEqual(wrong, []);
178});
Diagnostics6 git calls · 35 ms
CallStatusSizeTime035 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=11990 git.response.cache=UPDATING git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetch2005.1 KiB · hit25 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=24 git.response.bytes=5231 git.response.cache=HIT git.response.duration_ms=25 git.fetch.wants=90fbe59be240290a496bd41a49b28ba5627c9e24 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5231 git.pkt.payload.max=5204 git.sideband.band1.bytes=5204 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e173182d7c6102
- Request
- 694f91e9…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format