GitHubivarvong

ply

main
ply6 git calls91 ms32 KiB4 cachedprotocol v2
505 lines34 KiBRaw
1// readLog (#56) against git as the oracle, over git http-backend. The space is a product of tagged unions:
2//
3// server tree0 (git's default) | blobnone (Artifacts-like: every filter but blob:none is a 400, test/proxy.js)
4// shape linear | merges | root (root within every limit but 1) | exact (first-parent length = n) | longer
5// limit 1 | n | big (larger than every history but `longer`)
6// behaviour honest | omit-shallow (the shallow-info section dropped) | foreign (one commit that is not an ancestor
7// of the want, added by asking upload-pack for it as a second want)
8//
9// Every cell's expected outcome comes from git alone: the commits from `git rev-list --first-parent -n <limit>` and
10// `git log` / `git cat-file` for each, `truncated` from `git rev-list --first-parent --count`, and whether a cut graph
11// exists (some commit at depth ≥ limit, from `git rev-list --parents`) for the omit-shallow cells: with a cut, dropping
12// the shallow lines is a withheld parent (`pack_corrupt`, #37); without one there are no shallow lines to drop and the
13// answer is git's. A foreign commit is always `pack_corrupt`.
14import { test, beforeAll as before, afterAll as after } from 'vitest';
15import assert from 'node:assert/strict';
16import { join } from 'node:path';
17import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, initRepo } from './server.js';
18import { startProxy } from './proxy.js';
19import { readLog, logWindow, logOn } from '../src/log.js';
20import { fetchPackFiltered } from '../src/fetchpack.js';
21import { isFilterRefusal, isFilterRejection } from '../src/fetch.js';
22import { listTree } from '../src/read.js';
23import { createTransport as createTransportR } from '../src/http.js';
24import { TYPE } from '../src/objects.js';
25import { DEFAULT_LIMITS } from '../src/limits.js';
26import { pkt } from '../src/pktline.js';
27import { concat } from '../src/bytes.js';
28import { Tracer, arrayExporter, flattenLine } from '../src/otel.js';
29import { unwrap, unwrapAsync } from './unwrap.js';
31// Throwing adapters over the Result API (0.2.0): the assertions below are about the throw.
32const createTransport = (...a) => unwrap(createTransportR(...a));
33const REJECT = 'Unsupported Git fetch filter; only blob:none is supported.';
34const N = 5, BIG = 64;
35const LIMITS = { 1: 1, n: N, big: BIG };
36const SHAPES = ['linear', 'merges', 'root', 'exact', 'longer'];
37const SERVERS = ['tree0', 'blobnone'];
38const BEHAVIOURS = ['honest', 'omit-shallow', 'foreign'];
39const APIS = ['readLog', 'logOn'];
41let root, cleanup, srv;
42before(async () => { ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root); });
43after(async () => { await srv.close(); cleanup(); });
45const spansOf = ex => ex.lines.flatMap(flattenLine);
46const filterOf = body => /filter ([^\n]+)\n/.exec(Buffer.from(body).toString('latin1'))?.[1] ?? null;
48// ---- histories, built with plumbing at fixed dates ----
49function builder(bare) {
50 let clock = 1_700_000_000, k = 0;
51 return (parents, msg = null, { date = null, author = ['A U Thor', 'a@x'] } = {}) => {
52 const n = k++; const t = date ?? (clock += 60);
53 const blob = gitBuf(bare, ['hash-object', '-w', '--stdin'], Buffer.from(`content ${n}\n`)).toString().trim();
54 const tree = gitBuf(bare, ['mktree'], `100644 blob ${blob}\tf${n % 3}.txt\n`).toString().trim();
55 const env = { ...process.env, GIT_AUTHOR_NAME: author[0], GIT_AUTHOR_EMAIL: author[1], GIT_AUTHOR_DATE: `${t} +0130`, GIT_COMMITTER_NAME: 'C', GIT_COMMITTER_EMAIL: 'c@x', GIT_COMMITTER_DATE: `${t + 7} -0800` };
56 return git(bare, ['commit-tree', tree, ...parents.flatMap(p => ['-p', p]), '-m', msg ?? `commit ${n}`], { env }).trim();
57 };
59const chain = (mk, from, len, tag) => { let c = from; for (let i = 0; i < len; i++) c = mk(c ? [c] : [], `${tag} ${i}`); return c; };
61function buildShape(bare, shape) {
62 const mk = builder(bare);
63 let head;
64 if (shape === 'linear') head = chain(mk, null, 12, 'linear');
65 if (shape === 'merges') {
66 // first-parent length 12: two --no-ff merges of 3-commit branches (dated after the main line, so a date-ordered walk
67 // would interleave them) and an octopus; one non-ASCII author, one multi-line message
68 let c = chain(mk, null, 3, 'base');
69 const fork1 = c; c = chain(mk, c, 2, 'main');
70 const side1 = chain(mk, fork1, 3, 'side1');
71 c = mk([c, side1], "Merge branch 'side1'\n\nbody line\n indented\n");
72 const s2 = mk([c], 'oct a'), s3 = mk([c], 'oct b', { author: ['Zoë Ångström 李雷', 'zoë@exämple.org'] });
73 c = mk([mk([c], 'pre-octopus'), s2, s3], 'octopus');
74 c = chain(mk, c, 2, 'main2');
75 const side2 = chain(mk, c, 3, 'side2');
76 c = mk([mk([c], 'main3'), side2], "Merge branch 'side2'");
77 head = c;
78 }
79 if (shape === 'root') { const r0 = mk([], 'root'); const side = mk([r0], 'side'); head = mk([mk([r0], 'one'), side], 'merge side'); } // first-parent 3
80 if (shape === 'exact') { // first-parent length N; a side branch longer than N, so the window is cut on the side
81 const r0 = mk([], 'root'); const side = chain(mk, r0, N + 3, 'long side');
82 let c = chain(mk, r0, N - 2, 'main'); head = mk([c, side], 'merge long side');
83 }
84 if (shape === 'longer') { let c = null; for (let i = 0; i < 80; i++) { c = mk(c ? [c] : [], `l ${i}`); if (i % 10 === 9) c = mk([c, mk([c], `topic ${i}`)], `merge topic ${i}`); } head = c; }
85 git(bare, ['update-ref', 'refs/heads/main', head]);
86 git(bare, ['symbolic-ref', 'HEAD', 'refs/heads/main']);
87 // the foreign commit: a child of head, so a pack that carries it is the honest window plus one commit
88 git(bare, ['update-ref', 'refs/heads/foreign', mk([head], 'foreign')]);
89 return head;
92// ---- the oracle: git only ----
93const SEP = '\x1f', REC = '\x1e';
94const FMT = ['%H', '%T', '%P', '%an', '%ae', '%at', '%ad', '%cn', '%ce', '%ct', '%cd'].join(SEP);
95function oracle(bare, head, limit) {
96 const env = { ...process.env, TZ: 'UTC' };
97 const raw = git(bare, ['log', '--first-parent', '-n', String(limit), '--date=raw', `--format=${FMT}${REC}`, head], { env }).split(`${REC}\n`).filter(Boolean);
98 const iso = git(bare, ['log', '--first-parent', '-n', String(limit), '--date=iso-strict-local', `--format=%ad${SEP}%cd${REC}`, head], { env }).split(`${REC}\n`).filter(Boolean);
99 const commits = raw.map((line, i) => {
100 const [oid, tree, parents, an, ae, at, ad, cn, ce, ct, cd] = line.split(SEP);
101 const [aiso, ciso] = iso[i].split(SEP).map(d => d.replace(/\+00:00$/, 'Z'));
102 const body = gitBuf(bare, ['cat-file', 'commit', oid]).toString('utf8');
103 return { oid, tree, parents: parents ? parents.split(' ') : [],
104 author: { name: an, email: ae, time: aiso, epoch: Number(at), tz: ad.split(' ')[1] },
105 committer: { name: cn, email: ce, time: ciso, epoch: Number(ct), tz: cd.split(' ')[1] },
106 message: body.slice(body.indexOf('\n\n') + 2), signed: /^gpgsig(-sha256)? /m.test(body.slice(0, body.indexOf('\n\n'))) };
107 });
108 assert.deepEqual(commits.map(c => c.oid), git(bare, ['rev-list', '--first-parent', '-n', String(limit), head]).trim().split('\n'), 'git log and git rev-list agree');
109 const fpCount = Number(git(bare, ['rev-list', '--first-parent', '--count', head]).trim());
110 // depth from head (0-based, shortest path) over the whole history: a commit at depth ≥ limit is outside the window, so
111 // the window is cut and an honest server sends shallow lines
112 const parentsOf = new Map(git(bare, ['rev-list', '--parents', head]).trim().split('\n').map(l => { const [c, ...ps] = l.split(' '); return [c, ps]; }));
113 const depth = new Map([[head, 0]]); const q = [head];
114 for (let i = 0; i < q.length; i++) for (const p of parentsOf.get(q[i])) if (!depth.has(p)) { depth.set(p, depth.get(q[i]) + 1); q.push(p); }
115 return { commits, truncated: fpCount > limit, cut: [...depth.values()].some(d => d >= limit), fpCount };
118// ---- the server's behaviour, in a proxy in front of http-backend ----
119// Drop the shallow-info section (its header, lines and the delim after it) from a v2 fetch response.
120function omitShallow(bytes) {
121 const out = []; let o = 0, skipping = false;
122 while (o < bytes.length) {
123 const len = parseInt(Buffer.from(bytes.subarray(o, o + 4)).toString('latin1'), 16);
124 const end = o + (len < 4 ? 4 : len);
125 const payload = len >= 4 ? Buffer.from(bytes.subarray(o + 4, end)).toString('latin1') : null;
126 if (payload === 'shallow-info\n') skipping = true;
127 else if (skipping && len === 1) { skipping = false; o = end; continue; }
128 if (!skipping) out.push(bytes.subarray(o, end));
129 if (payload === 'packfile\n') { out.push(bytes.subarray(end)); break; }
130 o = end;
131 }
132 return concat(out);
134// Ask upload-pack for one more want than the client did (inserted after the client's first want).
135async function withForeign(url, reqBody, foreign) {
136 const s = Buffer.from(reqBody).toString('latin1'); const at = s.indexOf('want ');
137 const cut = at - 4 + parseInt(s.slice(at - 4, at), 16);
138 const body = concat([reqBody.subarray(0, cut), unwrap(pkt(`want ${foreign}\n`)), reqBody.subarray(cut)]);
139 const res = await fetch(`${url}/git-upload-pack`, { method: 'POST', body, headers: { 'content-type': 'application/x-git-upload-pack-request', 'git-protocol': 'version=2' } });
140 return new Uint8Array(await res.arrayBuffer());
142function serverFault(state) {
143 return ctx => {
144 if (ctx.stage !== 'fetch') return null;
145 const narrow = filterOf(ctx.reqBody) !== null && filterOf(ctx.reqBody) !== 'blob:none';
146 if (state.server === 'blobnone' && narrow) return ctx.phase === 'request' ? null : { type: 'status', status: 400, contentType: 'text/plain; charset=utf-8', body: REJECT };
147 if (state.behaviour === 'foreign' && ctx.phase === 'request') return { type: 'bytes', noForward: true, name: 'foreign', bytes: withForeign(state.upstream, ctx.reqBody, state.foreign) };
148 if (state.behaviour === 'omit-shallow' && ctx.phase !== 'request' && ctx.status === 200) return { type: 'bytes', name: 'omit-shallow', bytes: omitShallow(ctx.bytes) };
149 return ctx.phase === 'request' ? null : { type: 'pass' };
150 };
153test('logWindow (pure): first parent, truncated, the deepen contract', () => {
154 const C = (...parents) => ({ parents });
155 const a = 'a'.repeat(40), b = 'b'.repeat(40), c = 'c'.repeat(40), d = 'd'.repeat(40), x = 'e'.repeat(40);
156 const m = new Map([[a, C(b, c)], [b, C(d)], [c, C(d)], [d, C()]]);
157 assert.deepEqual(unwrap(logWindow(m, [], a, 1)), { chain: [a], truncated: true, cut: false });
158 assert.deepEqual(unwrap(logWindow(m, [], a, 3)), { chain: [a, b, d], truncated: false, cut: false });
159 assert.deepEqual(unwrap(logWindow(m, [], a, 9)), { chain: [a, b, d], truncated: false, cut: false });
160 const cutAtB = new Map([[a, C(b, c)], [b, C(d)], [c, C(d)]]);
161 assert.deepEqual(unwrap(logWindow(cutAtB, [b, c], a, 5)), { chain: [a, b], truncated: true, cut: true }, 'a server-side shallow boundary');
162 assert.throws(() => unwrap(logWindow(cutAtB, [c], a, 5)), e => e.code === 'pack_corrupt' && /withheld and .* not reported shallow/.test(e.message));
163 assert.throws(() => unwrap(logWindow(new Map([...m, [x, C()]]), [], a, 5)), e => e.code === 'pack_corrupt' && /not an ancestor/.test(e.message));
164 assert.throws(() => unwrap(logWindow(new Map([[b, C()]]), [], a, 5)), e => e.code === 'pack_corrupt' && /not in the pack/.test(e.message));
165});
167test('readLog: server × shape × limit × behaviour, every cell against git', async () => {
168 const state = { server: 'tree0', behaviour: 'honest', upstream: null, foreign: null };
169 const proxy = await startProxy(srv.url, serverFault(state));
170 const tally = { cells: 0, equal: 0, pack_corrupt: 0 };
171 try {
172 for (const shape of SHAPES) {
173 const name = `log-${shape}`; const bare = makeBareRepo(root, name);
174 const head = buildShape(bare, shape);
175 state.upstream = `${srv.url}/${name}.git`; state.foreign = git(bare, ['rev-parse', 'foreign']).trim();
176 const url = `${proxy.url}/${name}.git`;
177 for (const [lname, limit] of Object.entries(LIMITS)) {
178 const want = oracle(bare, head, limit);
179 for (const server of SERVERS) for (const behaviour of BEHAVIOURS) for (const api of APIS) {
180 Object.assign(state, { server, behaviour });
181 const cell = `${api} × ${server} × ${shape} × limit ${lname} (${limit}) × ${behaviour}`;
182 const fetchesBefore = proxy.log.filter(l => l.stage === 'fetch').length;
183 const ex = arrayExporter(); const quirks = {};
184 const tracer = new Tracer({ exporter: ex });
185 const corrupt = behaviour === 'foreign' || (behaviour === 'omit-shallow' && want.cut);
186 tally.cells++;
187 if (api === 'readLog') {
188 const run = unwrapAsync(readLog(url, { ref: 'main', limit }, { tracer, quirks }));
189 if (corrupt) {
190 await assert.rejects(run, e => e.code === 'pack_corrupt' && (behaviour === 'foreign' ? /not an ancestor/ : /not reported shallow/).test(e.message), cell);
191 tally.pack_corrupt++;
192 } else {
193 const got = await run;
194 assert.deepEqual(got, { ref: 'refs/heads/main', oid: head, commits: want.commits, truncated: want.truncated }, cell);
195 const root = spansOf(ex).find(s => s.name === 'git.read_log').attrs;
196 assert.equal(root['git.read_log.filter'], server === 'tree0' ? 'tree:0' : 'blob:none', cell);
197 assert.equal(root['git.read_log.commits'], want.commits.length, cell);
198 assert.equal(root['git.read_log.cut'], false, cell);
199 tally.equal++;
200 }
201 } else {
202 const got = await logOn(createTransport(url, { tracer, quirks }), { commit: head, limit });
203 if (corrupt) {
204 assert.equal(got.ok, false, cell);
205 assert.equal(got.error.code, 'pack_corrupt', cell);
206 assert.match(got.error.message, behaviour === 'foreign' ? /not an ancestor/ : /not reported shallow/, cell);
207 tally.pack_corrupt++;
208 } else {
209 assert.equal(got.ok, true, cell);
210 assert.deepEqual(got.value.commits, want.commits, cell);
211 assert.equal(got.value.truncated, want.truncated, cell);
212 assert.equal(got.value.cut, false, cell);
213 assert.equal(got.value.filter, server === 'tree0' ? 'tree:0' : 'blob:none', cell);
214 tally.equal++;
215 }
216 }
217 // one fetch; on the blob:none server one 400 first (learned into quirks)
218 assert.equal(proxy.log.filter(l => l.stage === 'fetch').length - fetchesBefore, server === 'tree0' ? 1 : 2, `${cell}: fetches`);
219 if (server === 'blobnone') assert.equal(quirks.treeFilter, 'blob:none', cell);
220 }
221 }
222 }
223 } finally { await proxy.close(); }
224 assert.equal(tally.cells, APIS.length * SERVERS.length * SHAPES.length * Object.keys(LIMITS).length * BEHAVIOURS.length);
225 console.log(` ${tally.cells} cells: ${tally.equal} equal to git, ${tally.pack_corrupt} pack_corrupt as predicted`);
226});
228test('readLog on a shallow server repository: the log stops at the server\'s cut, truncated, as git log there does', async () => {
229 const src = makeBareRepo(root, 'log-src'); const head = buildShape(src, 'linear');
230 const bare = join(root, 'log-shallow.git');
231 git(root, ['clone', '-q', '--bare', '--depth', '3', `file://${src}`, bare]);
232 for (const [k, v] of [['uploadpack.allowFilter', 'true'], ['protocol.version', '2']]) git(bare, ['config', k, v]);
233 for (const server of SERVERS) {
234 const proxy = await startProxy(srv.url, serverFault({ server, behaviour: 'honest' }));
235 try {
236 const got = await unwrapAsync(readLog(`${proxy.url}/log-shallow.git`, { ref: 'main', limit: N }, { tracer: new Tracer() }));
237 const oids = git(bare, ['rev-list', '--first-parent', head]).trim().split('\n');
238 assert.deepEqual(got.commits.map(c => c.oid), oids, server);
239 assert.equal(got.commits.length, 3); assert.equal(got.truncated, true, 'history continues past the cut');
240 const win = await logOn(createTransport(`${proxy.url}/log-shallow.git`, { tracer: new Tracer() }), { commit: head, limit: N });
241 assert.equal(win.ok, true, server);
242 assert.deepEqual(win.value.commits.map(c => c.oid), oids, `${server} logOn`);
243 assert.equal(win.value.truncated, true);
244 assert.equal(win.value.cut, true, `${server}: the chain stopped before the limit, on the server's shallow line`);
245 assert.equal(win.value.filter, server === 'tree0' ? 'tree:0' : 'blob:none');
246 } finally { await proxy.close(); }
247 }
248});
250test('readLog: refs, not_found, bad_request; never a TypeError', async () => {
251 const bare = makeBareRepo(root, 'log-refs'); const head = buildShape(bare, 'root');
252 git(bare, ['tag', '-a', '-m', 'annotated', 'v1', head]);
253 const url = `${srv.url}/log-refs.git`; const opts = () => ({ tracer: new Tracer() });
254 for (const [ref, name] of [[null, 'refs/heads/main'], ['HEAD', 'refs/heads/main'], ['v1', 'refs/tags/v1'], [head, head]]) {
255 const r = await unwrapAsync(readLog(url, { ref, limit: 2 }, opts()));
256 assert.equal(r.ref, name); assert.equal(r.oid, head); assert.equal(r.commits.length, 2); assert.equal(r.truncated, true);
257 }
258 assert.equal((await unwrapAsync(readLog(url, undefined, opts()))).commits.length, 3, 'default limit 20 > history');
259 const code = c => e => e.code === c && !(e instanceof TypeError);
260 await assert.rejects(unwrapAsync(readLog(url, { ref: 'nope' }, opts())), code('not_found'));
261 const absent = 'f'.repeat(40);
262 await assert.rejects(unwrapAsync(readLog(url, { ref: absent }, opts())), e => e.code === 'not_found' && e.miss?.kind === 'commit' && e.miss.miss.kind === 'absent' && e.miss.miss.oid === absent && !(e instanceof TypeError));
263 const treeOid = git(bare, ['rev-parse', 'main^{tree}']).trim();
264 await assert.rejects(unwrapAsync(readLog(url, { ref: treeOid }, opts())), e => e.code === 'not_found' && e.message === `${treeOid} is not a commit (tree)` && e.miss?.kind === 'commit' && e.miss.miss.kind === 'not_commit' && e.miss.miss.type === 'tree' && !(e instanceof TypeError));
265 for (const limit of [0, -1, 1.5, '5', NaN, Infinity, null, 1025, 2 ** 53]) await assert.rejects(unwrapAsync(readLog(url, { limit }, opts())), code('bad_request'), String(limit));
266 const limitMsg = 'readLog: limit must be an integer from 1 to 1024 (limits.maxMergeDepth)';
267 await assert.rejects(unwrapAsync(readLog(url, { limit: 0 }, opts())), e => e.message === limitMsg);
268 await assert.rejects(unwrapAsync(readLog(url, { limit: 1025 }, opts())), e => e.message === limitMsg);
269 assert.equal((await unwrapAsync(readLog(url, { limit: 1023 }, opts()))).commits.length, 3, 'one under the default cap');
270 assert.equal((await unwrapAsync(readLog(url, { limit: 1024 }, opts()))).commits.length, 3, 'the default cap itself');
271 for (const bad of [null, 5, 'main']) await assert.rejects(unwrapAsync(readLog(url, bad, opts())), code('bad_request'), String(bad));
272 await assert.rejects(unwrapAsync(readLog(url, {}, null)), code('bad_request'));
273 // no tracer is fine (operation() drops the spans); a tracer that is not one is bad_request
274 assert.equal((await readLog(url, {}, {})).ok, true, 'no tracer');
275 await assert.rejects(unwrapAsync(readLog(url, {}, { tracer: {} })), code('bad_request'), 'not a tracer');
276 await assert.rejects(unwrapAsync(readLog(url, { path: 'f0.txt' }, opts())), code('bad_request'), 'path is not supported');
277 await assert.rejects(unwrapAsync(readLog(url, { ref: 'a..b' }, opts())), code('bad_request'));
278 await assert.rejects(unwrapAsync(readLog(url, { ref: 7 }, opts())), code('bad_request'));
279 assert.equal((await unwrapAsync(readLog(url, { limit: 1 }, { tracer: new Tracer(), limits: { ...(await import('../src/limits.js')).DEFAULT_LIMITS, maxMergeDepth: 1 } }))).commits.length, 1);
280});
282test('readLog on blob:none: the trees are harvested; a following listTree on the same transport costs no fetch', async () => {
283 const bare = makeBareRepo(root, 'log-harvest'); buildShape(bare, 'merges');
284 const proxy = await startProxy(srv.url, serverFault({ server: 'blobnone', behaviour: 'honest' }));
285 try {
286 const tracer = new Tracer(); const t = createTransport(`${proxy.url}/log-harvest.git`, { tracer });
287 const log = await unwrapAsync(readLog(null, { ref: 'main', limit: N }, { tracer, transport: t }));
288 const fetches = () => proxy.log.filter(l => l.stage === 'fetch').length;
289 const before = fetches();
290 for (const c of log.commits) {
291 const ls = await unwrapAsync(listTree(null, { ref: c.oid }, { tracer, transport: t }));
292 assert.equal(ls.tree, c.tree); assert.deepEqual(ls.entries.map(e => e.oid), git(bare, ['ls-tree', '--format=%(objectname)', c.oid]).trim().split('\n'));
293 }
294 assert.equal(fetches(), before, 'every commit and root tree came from the harvest');
295 const head = git(bare, ['rev-parse', 'main']).trim();
296 const t2 = createTransport(`${proxy.url}/log-harvest.git`, { tracer, quirks: {} });
297 const beforeLog = fetches();
298 const win = await logOn(t2, { commit: head, limit: N });
299 assert.equal(win.ok, true);
300 assert.equal(fetches() - beforeLog, 2, 'logOn learns blob:none on its own transport');
301 const beforeTrees = fetches();
302 for (const c of win.value.commits) {
303 const ls = await unwrapAsync(listTree(null, { ref: c.oid }, { tracer, transport: t2 }));
304 assert.equal(ls.tree, c.tree);
305 }
306 assert.equal(fetches(), beforeTrees, 'logOn harvested the trees');
307 } finally { await proxy.close(); }
308});
310// Wide trees, curl-shaped: 48 directories of 80 files, 12 commits each changing one file. Bytes on the wire for a
311// 10-commit window under each filter (the fetch that answered; the blob:none figure excludes the 400).
312test('bytes: a 10-commit window on a wide repository, tree:0 vs blob:none', async () => {
313 const bare = makeBareRepo(root, 'log-wide'); const work = join(root, 'wwide');
314 const { mkdirSync, writeFileSync } = await import('node:fs');
315 mkdirSync(work); initRepo(work);
316 for (let d = 0; d < 48; d++) { mkdirSync(join(work, `dir${d}`)); for (let f = 0; f < 80; f++) writeFileSync(join(work, `dir${d}`, `file-with-a-longish-name-${f}.c`), `/* ${d}/${f} */\n`); }
317 const id = ['-c', 'user.name=w', '-c', 'user.email=w@x'];
318 git(work, ['add', '-A']); git(work, [...id, 'commit', '-q', '-m', 'seed']);
319 for (let i = 0; i < 12; i++) { writeFileSync(join(work, `dir${(i * 7) % 48}`, 'file-with-a-longish-name-0.c'), `/* change ${i} */\n`); git(work, [...id, 'commit', '-q', '-am', `change ${i}`]); }
320 git(work, ['push', '-q', bare, 'HEAD:refs/heads/main']);
321 const measure = async server => {
322 const proxy = await startProxy(srv.url, serverFault({ server, behaviour: 'honest' }));
323 try {
324 const quirks = server === 'blobnone' ? { treeFilter: 'blob:none' } : {};
325 const ex = arrayExporter();
326 const r = await unwrapAsync(readLog(`${proxy.url}/log-wide.git`, { ref: 'main', limit: 10 }, { tracer: new Tracer({ exporter: ex }), quirks }));
327 assert.deepEqual(r.commits.map(c => c.oid), git(bare, ['rev-list', '--first-parent', '-n', '10', 'main']).trim().split('\n'));
328 const f = spansOf(ex).filter(s => s.name === 'git.upload-pack fetch');
329 assert.equal(f.length, 1);
330 return { pack: f[0].attrs['git.pack.bytes'], wire: f[0].attrs['git.response.bytes'], objects: f[0].attrs['git.pack.objects'] };
331 } finally { await proxy.close(); }
332 };
333 const t0 = await measure('tree0'), bn = await measure('blobnone');
334 console.log(` 10 commits, 48 dirs × 80 files: tree:0 ${t0.pack} B pack / ${t0.wire} B response, ${t0.objects} objects; blob:none ${bn.pack} B pack / ${bn.wire} B response, ${bn.objects} objects (${(bn.pack / t0.pack).toFixed(1)}×)`);
335 assert.equal(t0.objects, 10, 'tree:0: the commits only');
336 assert.ok(bn.objects > 10 + 48 && bn.pack > t0.pack);
337});
339const PHRASES = [
340 ['only blob:none is supported', true],
341 ['Unsupported Git fetch filter; only blob:none is supported.', true],
342 ['ONLY BLOB:NONE IS SUPPORTED', true],
343 ['unsupported fetch filter', true],
344 ['unsupported git fetch filter', true],
345 ['filter sparse:oid=abc not supported', true],
346 ['filter blob:limit=1 not allowed', true],
347 ['', false],
348 ['blob:none', false],
349 ['filter ok', false],
350];
351// 400 is the boundary. 0 and 1 are below any HTTP status a server sends; 399/401 bracket it; 500 and 2048 are past it.
352const REFUSAL_STATUSES = [0, 1, 399, 400, 401, 500, 2048];
354test('isFilterRefusal: status 400 and the phrase, and nothing else isFilterRejection treats as a raw 400', () => {
355 for (const status of REFUSAL_STATUSES) for (const [text, phrase] of PHRASES) {
356 assert.equal(isFilterRefusal(status, text), status === 400 && phrase, `${status} × ${JSON.stringify(text)}`);
357 }
358 const raw = 'Unsupported Git fetch filter; only blob:none is supported.';
359 const wrapped = `HTTP 400 git-upload-pack: ${raw}`;
360 assert.equal(isFilterRejection({ code: 'http', status: 400, message: wrapped }), true);
361 assert.equal(isFilterRejection({ code: 'http', status: 400, message: raw }), true);
362 assert.equal(isFilterRejection({ code: 'http', status: 0, message: raw }), false);
363 assert.equal(isFilterRejection({ code: 'http', status: 1, message: raw }), false);
364 assert.equal(isFilterRejection({ code: 'http', status: 399, message: raw }), false);
365 assert.equal(isFilterRejection({ code: 'http', status: 401, message: raw }), false);
366 assert.equal(isFilterRejection({ code: 'http', status: 500, message: raw }), false);
367 assert.equal(isFilterRejection({ code: 'http', status: 2048, message: raw }), false);
368 assert.equal(isFilterRejection({ code: 'http', message: raw }), false, 'an http error with no status is not a 400');
369 assert.equal(isFilterRejection({ code: 'http', status: '400', message: raw }), false);
370 assert.equal(isFilterRejection({ code: 'remote_error', message: 'filter tree:0 not supported' }), true);
371 assert.equal(isFilterRejection({ code: 'protocol', message: 'unsupported fetch filter' }), true);
372 assert.equal(isFilterRejection({ code: 'remote_error', message: 'not our ref' }), false);
373 assert.equal(isFilterRejection({ code: 'protocol', message: '' }), false);
374 assert.equal(isFilterRejection({ code: 'pack_corrupt', status: 400, message: raw }), false);
375 assert.equal(isFilterRejection({ code: 'upstream_5xx', status: 500, message: raw }), false);
376 assert.equal(isFilterRejection(null), false);
377 assert.equal(isFilterRejection(undefined), false);
378});
380test('logOn limits and fetchPackFiltered arguments: 0, 1, one under the cap, the cap, one past it', async () => {
381 const bare = makeBareRepo(root, 'log-bounds');
382 const head = buildShape(bare, 'linear');
383 const url = `${srv.url}/log-bounds.git`;
384 const t = createTransport(url, { tracer: new Tracer() });
385 assert.equal((await logOn(t, null)).error.message, 'logOn: the second argument must be an object ({ commit, limit })');
386 assert.equal((await logOn(t, 'main')).error.message, 'logOn: the second argument must be an object ({ commit, limit })');
387 assert.equal((await logOn(t, {})).error.message, 'logOn: commit must be a string');
388 assert.equal((await logOn(t, { commit: 7, limit: 1 })).error.message, 'logOn: commit must be a string');
389 const capMsg = n => `logOn: limit must be an integer from 1 to ${n} (limits.maxMergeDepth)`;
390 assert.equal((await logOn(t, { commit: head, limit: 1025 })).error.message, capMsg(1024));
391 assert.equal((await logOn(t, { commit: head, limit: 0 })).error.message, capMsg(1024));
392 for (const limit of [-1, 1.5, '5', NaN, Infinity, null, 2 ** 53, 2048]) {
393 const got = await logOn(t, { commit: head, limit });
394 assert.equal(got.ok, false, String(limit));
395 assert.equal(got.error.code, 'bad_request', String(limit));
396 assert.equal(got.error.message, capMsg(1024), String(limit));
397 }
398 const narrow = createTransport(url, { tracer: new Tracer(), limits: { ...DEFAULT_LIMITS, maxMergeDepth: 4 } });
399 const fp = n => git(bare, ['rev-list', '--first-parent', '-n', String(n), head]).trim().split('\n');
400 for (const limit of [0, 5, 8, 64]) {
401 const got = await logOn(narrow, { commit: head, limit });
402 assert.equal(got.ok, false, String(limit));
403 assert.equal(got.error.message, capMsg(4), String(limit));
404 }
405 for (const limit of [1, 3, 4]) {
406 const got = await logOn(narrow, { commit: head, limit });
407 assert.equal(got.ok, true, String(limit));
408 assert.deepEqual(got.value.commits.map(c => c.oid), fp(limit), String(limit));
409 assert.equal(got.value.truncated, true, String(limit));
410 assert.equal(got.value.cut, false, String(limit));
411 }
412 const omitted = await logOn(narrow, { commit: head });
413 assert.equal(omitted.ok, false, 'the default 20 is past this cap of 4');
414 assert.equal(omitted.error.message, capMsg(4));
415 const full = await logOn(t, { commit: head });
416 assert.equal(full.ok, true);
417 assert.equal(full.value.commits.length, 12, 'default limit 20 holds this history of 12');
418 assert.equal(full.value.truncated, false);
420 assert.equal((await fetchPackFiltered(t, null)).error.message, 'fetchPackFiltered: expected { wants, filter, deepen }');
421 assert.equal((await fetchPackFiltered(t, undefined)).error.message, 'fetchPackFiltered: expected { wants, filter, deepen }');
422 assert.equal((await fetchPackFiltered(t, { wants: [head], filter: 'tree:1' })).error.message, 'fetchPackFiltered: filter must be tree:0 or blob:none, got "tree:1"');
423 assert.equal((await fetchPackFiltered(t, { wants: [head], filter: 0 })).error.message, 'fetchPackFiltered: filter must be tree:0 or blob:none, got 0');
424 assert.equal((await fetchPackFiltered(t, { wants: [head] })).error.message, 'fetchPackFiltered: filter must be tree:0 or blob:none, got undefined');
425 for (const filter of ['', 'blob:limit', 'TREE:0', null, false]) {
426 const got = await fetchPackFiltered(t, { wants: [head], filter });
427 assert.equal(got.ok, false, String(filter));
428 assert.equal(got.error.code, 'bad_request');
429 }
430 assert.equal((await fetchPackFiltered(t, { wants: head, filter: 'tree:0' })).error.message, 'fetchPackFiltered: wants must be a list of object ids');
431 assert.equal((await fetchPackFiltered(t, { wants: [], filter: 'tree:0' })).error.message, 'fetchPackFiltered: wants must name at least one object');
432 const parent = git(bare, ['rev-parse', `${head}^`]).trim();
433 const two = await fetchPackFiltered(t, { wants: [head, parent], filter: 'tree:0', deepen: 1 });
434 assert.equal(two.ok, true);
435 assert.equal(two.value.learned, false);
436 assert.equal(two.value.filter, 'tree:0');
437 assert.equal(two.value.objects.has(head), true);
438 assert.equal(two.value.objects.has(parent), true);
439 const neg = await fetchPackFiltered(t, { wants: [head], filter: 'tree:0', deepen: -1 });
440 assert.equal(neg.ok, false);
441 assert.equal(neg.error.code, 'bad_request');
442});
444test('fetchPackFiltered deepen 0/1/limit-1/limit/limit+1 against git rev-list, and the second fetch asks blob:none', async () => {
445 const bare = makeBareRepo(root, 'log-deepen');
446 const head = buildShape(bare, 'linear');
447 const all = Number(git(bare, ['rev-list', '--count', head]).trim());
448 assert.equal(all, 12);
449 const url = `${srv.url}/log-deepen.git`;
450 const t = createTransport(url, { tracer: new Tracer() });
451 const commitsOf = objects => [...objects.values()].filter(o => o.type === TYPE.COMMIT).map(o => o).length;
452 const fp = n => git(bare, ['rev-list', '--first-parent', '-n', String(n), head]).trim().split('\n');
453 // deepen 0 is omitted on the wire (fetchBody), so git sends the whole history. 12 is this repo's length.
454 for (const [deepen, n] of [[0, 12], [1, 1], [11, 11], [12, 12], [13, 12]]) {
455 const got = await fetchPackFiltered(t, { wants: [head], filter: 'tree:0', deepen });
456 assert.equal(got.ok, true, String(deepen));
457 assert.equal(got.value.learned, false, String(deepen));
458 assert.equal(got.value.filter, 'tree:0', String(deepen));
459 assert.equal(commitsOf(got.value.objects), n, `deepen ${deepen}`);
460 if (deepen !== 0) assert.deepEqual([...got.value.objects.keys()].filter(oid => fp(n).includes(oid)).length, n);
461 }
462 const wide = await fetchPackFiltered(t, { wants: [head], filter: 'blob:none', deepen: 1 });
463 assert.equal(wide.ok, true);
464 assert.equal(wide.value.learned, false, 'asking for blob:none is not a fallback');
465 assert.equal(wide.value.filter, 'blob:none');
466 assert.ok([...wide.value.objects.values()].some(o => o.type === TYPE.TREE), 'blob:none still carries the trees');
468 const filters = [];
469 const proxy = await startProxy(srv.url, ctx => {
470 if (ctx.stage !== 'fetch') return null;
471 const filter = filterOf(ctx.reqBody);
472 if (ctx.phase === 'request') filters.push(filter);
473 if (ctx.phase === 'request') return null;
474 if (filter !== null && filter !== 'blob:none') return { type: 'status', status: 400, contentType: 'text/plain; charset=utf-8', body: REJECT };
475 return { type: 'pass' };
476 });
477 try {
478 const quirks = {};
479 const ex1 = arrayExporter();
480 const first = await fetchPackFiltered(createTransport(`${proxy.url}/log-deepen.git`, { tracer: new Tracer({ exporter: ex1 }), quirks }), { wants: [head], filter: 'tree:0', deepen: 1 });
481 assert.equal(first.ok, true);
482 assert.equal(first.value.learned, true);
483 assert.equal(first.value.filter, 'blob:none');
484 assert.equal(quirks.treeFilter, 'blob:none');
485 assert.deepEqual(filters, ['tree:0', 'blob:none']);
486 const learned = spansOf(ex1).filter(s => s.name === 'git.upload-pack fetch').at(-1);
487 assert.equal(learned.attrs['git.fetch.filter.learned'], true);
488 filters.length = 0;
489 const ex2 = arrayExporter();
490 const second = await fetchPackFiltered(createTransport(`${proxy.url}/log-deepen.git`, { tracer: new Tracer({ exporter: ex2 }), quirks }), { wants: [head], filter: 'tree:0', deepen: 1 });
491 assert.equal(second.ok, true);
492 assert.equal(second.value.learned, false);
493 assert.equal(second.value.filter, 'blob:none');
494 assert.deepEqual(filters, ['blob:none'], 'the learned transport does not ask for tree:0 again');
495 const direct = spansOf(ex2).find(s => s.name === 'git.upload-pack fetch');
496 assert.equal(direct.attrs['git.fetch.filter.learned'], false);
497 assert.equal(direct.attrs['git.fetch.filter.fallback'], 'blob:none');
498 assert.equal(second.value.objects.has(head), true);
499 const again = await logOn(createTransport(`${proxy.url}/log-deepen.git`, { tracer: new Tracer(), quirks }), { commit: head, limit: 1 });
500 assert.equal(again.ok, true);
501 assert.equal(again.value.filter, 'blob:none');
502 assert.equal(again.value.commits.length, 1);
503 assert.deepEqual(filters, ['blob:none', 'blob:none']);
504 } finally { await proxy.close(); }
505});
Diagnostics6 git calls · 91 ms
CallStatusSizeTime091 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=79
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2009.3 KiB80 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=9535
git.response.cache=MISS
git.response.duration_ms=80
git.fetch.wants=d5bb8e59086c9b7020b896e046be7e7898ad1dc4
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=9535
git.pkt.payload.max=9508
git.sideband.band1.bytes=9508
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0faa51a65332c
Request
dba0eaf4…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format