Artifactspublic-v0

ply

main
ply4 git calls27 ms43 KiB3 cachedprotocol v2
178 lines14 KiBRaw
1// Issue #46: the blob:none fallback applies the caller's limits to the objects the caller asked for, never to the
2// extras the widening brought. One sibling directory over maxTreeEntries used to fail every read on a blob:none-only
3// host (Cloudflare Artifacts: curl's tests/data, 2,135 entries, under the viewer's 2,000) while a tree:0 server answered.
4//
5// The space, as tagged unions, and every cell of their product (5 × 3 × 4 × 2 = 120), each on a fresh copy of the repo:
6// server: tree0 (git http-backend as is) | blobnone (a proxy that 400s every filter but blob:none, as Artifacts does)
7// wanted: small (a tree under the limit: small/) | big (a tree over the limit: big/) | blob (top.txt; only the root
8// tree is on the way to it)
9// sibling: none | over (huge/, over the limit) | malformed (bad/, entries out of order: git fsck refuses it)
10// | many (huge1..3 over the limit beside ok1, ok2 under it)
11// op: readFile | listTree | editFile | checkoutTree | merge (mergeBranch: the merge-base walk, fetchPackTree0,
12// then the tree merge), each pointed at `wanted` (a file in it, the directory itself, or edits in it)
13// Expected outcome per cell (`expected` below):
14// - a malformed sibling on blobnone: pack_corrupt (#18/#37: a server that sent it is not trusted), except
15// checkoutTree, whose own blob:none pack parses only the trees it walks (on both servers);
16// - else wanted big: limit_exceeded (the wanted tree gets the caller's limits, also when it was skipped from an
17// earlier wide pack and is now fetched on its own);
18// - else listTree/checkoutTree of the file top.txt: not_found;
19// - else ok, and git agrees (cat-file, ls-tree, the edited blob, merge-tree --write-tree).
20// Wherever the sibling is not malformed, tree0 and blobnone give the same answer, oid for oid.
21import { test, beforeAll as before, afterAll as after } from 'vitest';
22import assert from 'node:assert/strict';
23import { join } from 'node:path';
24import { cpSync } from 'node:fs';
25import { createHash } from 'node:crypto';
26import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, fsckProblems } from './server.js';
27import { startProxy } from './proxy.js';
28import { editFile } from '../src/edit.js';
29import { readFile, listTree } from '../src/read.js';
30import { checkoutTree } from '../src/checkout.js';
31import { unwrapAsync } from './unwrap.js';
32import { mergeBranch } from '../src/merge.js';
33import { Tracer, arrayExporter, flattenLine } from '../src/otel.js';
34import { withLimits } from '../src/limits.js';
36const L = 10; // maxTreeEntries; every "over" directory holds L + 1 entries, the issue's core cell (limit = size − 1)
37const LIMITS = withLimits({ maxTreeEntries: L });
38const A = { name: 'w', email: 'w@x' };
39const TIME = 1_700_000_000;
40const ENV = { GIT_AUTHOR_NAME: 's', GIT_AUTHOR_EMAIL: 's@x', GIT_AUTHOR_DATE: `${TIME} +0000`, GIT_COMMITTER_NAME: 's', GIT_COMMITTER_EMAIL: 's@x', GIT_COMMITTER_DATE: `${TIME} +0000` };
41const REJECT = 'Unsupported Git fetch filter; only blob:none is supported.';
42const filterOf = body => /filter ([^\n]+)\n/.exec(Buffer.from(body).toString('latin1'))?.[1] ?? null;
43const artifactsFault = ctx => (ctx.stage === 'fetch' && filterOf(ctx.reqBody) && filterOf(ctx.reqBody) !== 'blob:none' ? { type: 'status', status: 400, contentType: 'text/plain; charset=utf-8', body: REJECT } : { type: 'pass' });
45export const SERVERS = ['tree0', 'blobnone'];
46export const WANTED = ['small', 'big', 'blob'];
47export const SIBLINGS = ['none', 'over', 'malformed', 'many'];
48export const OPS = ['readFile', 'listTree', 'editFile', 'checkoutTree', 'merge'];
50export function expected({ server, op, wanted, sibling }) {
51 if (sibling === 'malformed' && server === 'blobnone' && op !== 'checkoutTree') return 'pack_corrupt';
52 if (wanted === 'big') return 'limit_exceeded';
53 if (wanted === 'blob' && (op === 'listTree' || op === 'checkoutTree')) return 'not_found';
54 return 'ok';
56const overCount = ({ wanted, sibling }) => (wanted === 'big' ? 1 : 0) + ({ none: 0, over: 1, malformed: 0, many: 3 })[sibling];
58// ---- the repository, built with plumbing so both servers' copies (and every run) have the same oids ----
59const hashBlob = (bare, text) => gitBuf(bare, ['hash-object', '-w', '--stdin'], Buffer.from(text)).toString().trim();
60function writeTree(bare, spec) {
61 const lines = Object.entries(spec).map(([name, v]) => {
62 if (typeof v === 'string') return `100644 blob ${hashBlob(bare, v)}\t${name}`;
63 if (v.raw) return `040000 tree ${v.raw}\t${name}`;
64 return `040000 tree ${writeTree(bare, v)}\t${name}`;
65 });
66 return gitBuf(bare, ['mktree'], Buffer.from(lines.join('\n') + '\n')).toString().trim();
68// A tree git fsck refuses (treeNotSorted) but upload-pack serves: written --literally, entries b then a.
69function badTree(bare) {
70 const e = (name, oid) => Buffer.concat([Buffer.from(`100644 ${name}\0`), Buffer.from(oid, 'hex')]);
71 return gitBuf(bare, ['hash-object', '-t', 'tree', '--literally', '-w', '--stdin'], Buffer.concat([e('b', hashBlob(bare, 'b\n')), e('a', hashBlob(bare, 'a\n'))])).toString().trim();
73const commit = (bare, tree, parents, msg) => git(bare, ['commit-tree', tree, ...parents.flatMap(p => ['-p', p]), '-m', msg], { env: { ...process.env, ...ENV } }).trim();
74const dirOf = (tag, n = L + 1) => Object.fromEntries(Array.from({ length: n }, (_, i) => [`f${i}.txt`, `${tag}${i}\nmore\n`]));
76// The target, per `wanted`: the file each op reads or edits, the directory listTree/checkoutTree name, and the two
77// non-overlapping edits the merge's sides make.
78const TARGET = {
79 small: { file: 'small/a.txt', find: 'a1', dir: 'small', feat: s => { s.small['a.txt'] = 'a1 feat\na2\na3\n'; }, main: s => { s.small['b.txt'] = 'b main\n'; } },
80 big: { file: 'big/f0.txt', find: 'big0', dir: 'big', feat: s => { s.big['f0.txt'] = 'big0 feat\nmore\n'; }, main: s => { s.big['f1.txt'] = 'big1 main\nmore\n'; } },
81 blob: { file: 'top.txt', find: 'l1', dir: 'top.txt', feat: s => { s['top.txt'] = 'l1 feat\nl2\nl3\nl4\nl5\n'; }, main: s => { s['top.txt'] = 'l1\nl2\nl3\nl4\nl5 main\n'; } },
82};
83function spec(wanted, sibling, bad) {
84 const s = { 'top.txt': 'l1\nl2\nl3\nl4\nl5\n', small: { 'a.txt': 'a1\na2\na3\n', 'b.txt': 'b\n' } };
85 if (wanted === 'big') s.big = dirOf('big');
86 if (sibling === 'over') s.huge = dirOf('huge');
87 if (sibling === 'malformed') s.bad = { raw: bad };
88 if (sibling === 'many') { for (const k of [1, 2, 3]) s[`huge${k}`] = dirOf(`huge${k}_`); s.ok1 = { 'x.txt': 'x\n' }; s.ok2 = { 'y.txt': 'y\n' }; }
89 return s;
91// base C0; feat = C0 + the feat edit; main = C0 + the main edit (merge merges feat into main; the rest read main)
92function template(root, wanted, sibling) {
93 const name = `tpl-${wanted}-${sibling}`; const bare = makeBareRepo(root, name);
94 const bad = sibling === 'malformed' ? badTree(bare) : null;
95 const c0 = commit(bare, writeTree(bare, spec(wanted, sibling, bad)), [], 'base');
96 const f = spec(wanted, sibling, bad); TARGET[wanted].feat(f); const c1 = commit(bare, writeTree(bare, f), [c0], 'feat');
97 const m = spec(wanted, sibling, bad); TARGET[wanted].main(m); const c2 = commit(bare, writeTree(bare, m), [c0], 'main');
98 git(bare, ['update-ref', 'refs/heads/main', c2]); git(bare, ['update-ref', 'refs/heads/feat', c1]);
99 if (sibling === 'malformed') assert.match(fsckErr(bare), /treeNotSorted|not properly sorted/, 'git refuses the bad tree');
100 return bare;
102const fsckErr = bare => fsckProblems(bare).join('\n');
104// ---- one cell ----
105const sha = b => createHash('sha256').update(b).digest('hex').slice(0, 12);
106const drain = async stream => { const out = []; for await (const c of stream) out.push(Buffer.from(c)); return Buffer.concat(out); };
107async function runOp(op, url, wanted, opts) {
108 const T = TARGET[wanted];
109 if (op === 'readFile') { const r = await unwrapAsync(readFile(url, { path: T.file }, opts)); const b = await drain(r.stream); return { answer: `blob ${r.blob} ${sha(b)}`, check: bare => assert.ok(b.equals(gitBuf(bare, ['show', `main:${T.file}`]))) }; }
110 if (op === 'listTree') {
111 const r = await unwrapAsync(listTree(url, { path: T.dir }, opts));
112 return { answer: `tree ${r.tree} ${sha(JSON.stringify(r.entries))}`, check: bare => assert.deepEqual(r.entries.map(e => `${e.mode} ${e.type} ${e.oid}\t${e.name}`), git(bare, ['ls-tree', `main:${T.dir}`]).trim().split('\n')) };
113 }
114 if (op === 'editFile') {
115 const r = await unwrapAsync(editFile(url, { path: T.file, find: T.find, replace: `${T.find}X`, author: A, message: 'edit', time: TIME }, opts));
116 return { answer: `commit ${r.newCommit}`, check: bare => { assert.equal(git(bare, ['rev-parse', 'main']).trim(), r.newCommit); assert.ok(gitBuf(bare, ['show', `main:${T.file}`]).toString().startsWith(`${T.find}X`)); } };
117 }
118 if (op === 'checkoutTree') {
119 const r = await unwrapAsync(checkoutTree(url, { path: T.dir }, opts));
120 const files = []; for await (const f of r.files) files.push(`${f.path} ${f.oid} ${sha(await drain(f.stream))}`);
121 return { answer: `tree ${r.tree} ${sha(files.join('\n'))}`, check: bare => assert.deepEqual(r.entries.map(e => `${e.mode} blob ${e.oid}\t${e.path}`), git(bare, ['ls-tree', '-r', `main:${T.dir}`]).trim().split('\n')) };
122 }
123 const r = await unwrapAsync(mergeBranch(url, { ref: 'refs/heads/main', from: 'refs/heads/feat', author: A, message: 'merge', time: TIME }, opts));
124 return { answer: `${r.outcome} ${r.oid} ${r.tree}`, check: bare => { assert.equal(r.outcome, 'merge'); assert.equal(r.tree, git(bare, ['merge-tree', '--write-tree', `${r.oid}^1`, `${r.oid}^2`]).trim().split('\n')[0]); assert.equal(git(bare, ['rev-parse', 'main']).trim(), r.oid); } };
127let root, cleanup, srv, proxy, n = 0;
128const templates = new Map();
129before(async () => {
130 ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root); proxy = await startProxy(srv.url, artifactsFault);
131 for (const w of WANTED) for (const s of SIBLINGS) templates.set(`${w}/${s}`, template(root, w, s));
132});
133after(async () => { await proxy.close(); await srv.close(); cleanup(); });
135async function cell(c) {
136 const name = `c${n++}`; const bare = join(root, `${name}.git`);
137 cpSync(templates.get(`${c.wanted}/${c.sibling}`), bare, { recursive: true });
138 const ex = arrayExporter(); const quirks = {};
139 const url = `${c.server === 'tree0' ? srv.url : proxy.url}/${name}.git`;
140 let got, answer = null, err = null;
141 try { const r = await runOp(c.op, url, c.wanted, { tracer: new Tracer({ exporter: ex }), limits: LIMITS, quirks }); answer = r.answer; r.check(bare); got = 'ok'; }
142 catch (e) { if (e?.code === 'ERR_ASSERTION') throw e; err = e; got = e?.code ?? String(e); answer = `${e?.code}: ${String(e?.message).replace(/[0-9a-f]{40}/g, '<oid>')}`; }
143 const fetches = ex.lines.flatMap(flattenLine).filter(s => s.name === 'git.upload-pack fetch');
144 const wide = fetches.filter(s => s.attrs['git.fetch.filter.fallback'] === 'blob:none');
145 const skipped = wide.map(s => s.attrs['git.fetch.wide.skipped'] ?? 0);
146 return { ...c, got, answer, err, skipped, wide, learned: quirks.treeFilter ?? '' };
149test('#46 every cell of server × wanted × sibling × op: the caller\'s limits bind what it asked for, never the extras; tree:0 and blob:none agree', { timeout: 300_000 }, async t => {
150 const rows = [];
151 for (const op of OPS) for (const wanted of WANTED) for (const sibling of SIBLINGS) for (const server of SERVERS) rows.push(await cell({ server, op, wanted, sibling }));
152 const wrong = [];
153 const by = new Map(rows.map(r => [`${r.server}/${r.op}/${r.wanted}/${r.sibling}`, r]));
154 for (const r of rows) {
155 const want = expected(r);
156 if (r.got !== want) wrong.push(`${r.server} ${r.op} ${r.wanted} ${r.sibling}: expected ${want}, got ${r.answer}`);
157 if (r.server === 'blobnone' && r.op !== 'checkoutTree' && r.learned !== 'blob:none') wrong.push(`${r.op} ${r.wanted} ${r.sibling}: blob:none not learned`);
158 if (r.server === 'tree0' && r.wide.length) wrong.push(`tree0 ${r.op} ${r.wanted} ${r.sibling}: a blob:none fallback against a tree:0 server`);
159 // the skipped extras are on the span: exactly the over-limit trees in the first wide pack of a single-tip read, any
160 // in the walk's; none at all where there are none
161 if (r.server === 'blobnone' && r.op !== 'checkoutTree' && r.sibling !== 'malformed') {
162 const k = overCount(r), total = r.skipped.reduce((a, b) => a + b, 0);
163 if (['readFile', 'listTree', 'editFile'].includes(r.op) && r.skipped[0] !== k) wrong.push(`${r.op} ${r.wanted} ${r.sibling}: first wide fetch skipped ${r.skipped[0]}, expected ${k}`);
164 if (k === 0 && total !== 0) wrong.push(`${r.op} ${r.wanted} ${r.sibling}: skipped ${total} with nothing over the limit`);
165 if (k > 0 && !(total > 0 && r.wide.some(s => s.attrs['git.fetch.wide.skipped_reason'] === 'tree entries'))) wrong.push(`${r.op} ${r.wanted} ${r.sibling}: over-limit extras not counted as skipped (tree entries)`);
166 }
167 }
168 // parity: wherever the sibling is well formed, the two servers give the same answer (oids, bytes, error code)
169 for (const op of OPS) for (const wanted of WANTED) for (const sibling of SIBLINGS) {
170 if (sibling === 'malformed') continue;
171 const a = by.get(`tree0/${op}/${wanted}/${sibling}`), b = by.get(`blobnone/${op}/${wanted}/${sibling}`);
172 const key = x => (x.got === 'ok' ? x.answer : x.got);
173 if (key(a) !== key(b)) wrong.push(`parity ${op} ${wanted} ${sibling}: tree0 ${a.answer} | blobnone ${b.answer}`);
174 }
175 console.log(`cells ${rows.length}, as expected ${rows.length - new Set(wrong.filter(w => !w.startsWith('parity')).map(w => w.split(':')[0])).size}`);
176 for (const r of rows) console.log(`CELL ${r.op.padEnd(12)} ${r.wanted.padEnd(5)} ${r.sibling.padEnd(9)} ${r.server.padEnd(8)} expect ${expected(r).padEnd(14)} got ${r.got.padEnd(14)} skipped [${r.skipped}] ${r.got === 'ok' ? '' : r.answer.slice(0, 90)}`);
177 assert.deepEqual(wrong, []);
178});
Diagnostics4 git calls · 27 ms
CallStatusSizeTime027 ms
discover200152 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=38434
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=38434
git.pkt.payload.max=38339
git.sideband.band1.bytes=38338
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=63
git.pack.bytes=38338
git.pack.ofs_delta=0
git.pack.ref_delta=8
git.pack.chain.max=8
git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77
git.pack.full=55
Open pack map
fetch2005.1 KiB · hit23 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=23
git.response.bytes=5227
git.response.cache=HIT
git.response.duration_ms=23
git.fetch.wants=90fbe59be240290a496bd41a49b28ba5627c9e24
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5227
git.pkt.payload.max=5206
git.sideband.band1.bytes=5205
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2be0458f1cfc3
Request
82dc2765…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format