GitHubivarvong

ply

main
ply6 git calls93 ms27 KiB4 cachedprotocol v2
215 lines13 KiBRaw
1// The default depth (fetchObject sends `deepen 1` for a want that may be a commit) on the paths beside the tree:0 single
2// fetch: the blob:none fallback, the unfiltered fallback, the caller's `filter: ''`, the re-ask without `deepen` on a
3// server without `shallow`, and expectType tag. The oracle is git, as in fetchobject-deepen.test.js: a line of N commits
4// built by git fast-import, each changing the one file f.txt, served by git http-backend; every exchange is recorded
5// and the pack in it is counted from its own header and by git index-pack + git verify-pack -v.
6// Expected counts are hand-derived from that history (every commit has its own root tree and its own f.txt blob):
7// depth 1, blob:none 1 commit + its 1 tree = 2 (no depth: N commits + N trees = 2N)
8// depth 1, no filter 1 commit + its 1 tree + its 1 blob = 3 (no depth: N commits, N trees, N blobs = 3N)
9import { test, beforeAll as before, afterAll as after } from 'vitest';
10import assert from 'node:assert/strict';
11import { join } from 'node:path';
12import { mkdirSync, writeFileSync } from 'node:fs';
13import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer } from './server.js';
14import { fetchObject } from '../src/fetch.js';
15import { createTransport } from '../src/http.js';
16import { TYPE } from '../src/objects.js';
17import { unwrap } from './unwrap.js';
19const N = 120;
20let root, cleanup, srv, bare, url, stockUrl, tip, tag, k = 0;
22before(async () => {
23 ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root);
24 bare = makeBareRepo(root, 'hist'); url = `${srv.url}/hist.git`;
25 // git's defaults: no uploadpack.allowFilter, so `fetch=` lists no `filter` and upload-pack dies on one
26 const stock = makeBareRepo(root, 'stock', { stock: true }); stockUrl = `${srv.url}/stock.git`;
27 let stream = '';
28 for (let i = 1; i <= N; i++) {
29 const msg = `c${i}\n`, body = `line ${i}\n`;
30 stream += `commit refs/heads/main\nmark :${i}\ncommitter Ada <ada@x> ${1_700_000_000 + i} +0000\ndata ${msg.length}\n${msg}${i > 1 ? `from :${i - 1}\n` : ''}M 100644 inline f.txt\ndata ${body.length}\n${body}\n`;
31 }
32 // the same stream into both: the same objects, the same ids
33 for (const dir of [bare, stock]) gitBuf(dir, ['fast-import', '--quiet'], Buffer.from(stream));
34 git(bare, ['-c', 'user.name=Ada', '-c', 'user.email=ada@x', 'tag', '-a', '-m', 'v1', 'v1', 'main']);
35 tip = git(bare, ['rev-parse', 'main']).trim();
36 tag = git(bare, ['rev-parse', 'refs/tags/v1']).trim();
37 assert.equal(git(stock, ['rev-parse', 'main']).trim(), tip);
38 assert.equal(Number(git(bare, ['rev-list', '--count', 'main']).trim()), N);
39 // what the hand-derived counts rest on: N distinct root trees, N distinct blobs
40 assert.equal(new Set(git(bare, ['log', '--format=%T', 'main']).trim().split('\n')).size, N);
41 assert.equal(git(bare, ['cat-file', '--batch-all-objects', '--batch-check=%(objecttype)']).split('\n').filter(x => x === 'blob').length, N);
42 assert.equal(git(bare, ['cat-file', '-t', tag]).trim(), 'tag');
43});
44after(async () => { await srv.close(); cleanup(); });
46// A transport on `remote` whose every exchange is kept: the request body as text, the response bytes. `rewrite` may
47// answer in the server's place.
48function recording(remote, quirks = {}, rewrite = null) {
49 const log = [];
50 const fetchImpl = async (input, init = {}) => {
51 const body = init.body === undefined || init.body === null ? '' : Buffer.from(await new Response(init.body).arrayBuffer()).toString('latin1');
52 const method = init.method ?? 'GET';
53 const faked = rewrite === null ? null : await rewrite({ url: String(input), method, body });
54 const res = faked ?? await fetch(input, { ...init, body: body === '' ? undefined : Buffer.from(body, 'latin1') });
55 const bytes = Buffer.from(await res.arrayBuffer());
56 log.push({ url: String(input), method, body, bytes, status: res.status });
57 return new Response(bytes, { status: res.status, headers: res.headers });
58 };
59 const t = unwrap(createTransport(remote, { fetchImpl, quirks: { treeFilter: 'tree:0', blobWants: 'many', ...quirks } }));
60 return { t, log, fetches: () => log.filter(x => x.method === 'POST' && x.body.includes('command=fetch')) };
62// The band-1 bytes of a fetch response: the pack.
63function packOf(bytes) {
64 const parts = [];
65 let i = 0, inPack = false;
66 while (i + 4 <= bytes.length) {
67 const len = parseInt(bytes.subarray(i, i + 4).toString('latin1'), 16);
68 if (len < 4) { i += 4; continue; }
69 const payload = bytes.subarray(i + 4, i + len); i += len;
70 if (inPack) { if (payload[0] === 1) parts.push(payload.subarray(1)); continue; }
71 if (payload.toString('latin1').trimEnd() === 'packfile') inPack = true;
72 }
73 return Buffer.concat(parts);
75// The pack counted: by its header, and by git (index-pack accepts it, verify-pack lists its objects).
76function counted(bytes) {
77 const pack = packOf(bytes);
78 const dir = join(root, `count${k++}`); mkdirSync(dir); git(dir, ['init', '-q', '--bare', '.']);
79 const file = join(dir, 'x.pack'); writeFileSync(file, pack);
80 git(dir, ['index-pack', file]);
81 const by = { commit: 0, tree: 0, blob: 0, tag: 0 };
82 for (const l of git(dir, ['verify-pack', '-v', join(dir, 'x.idx')]).split('\n')) { const m = /^[0-9a-f]{40} (commit|tree|blob|tag) /.exec(l); if (m) by[m[1]]++; }
83 assert.equal(pack.subarray(0, 4).toString('latin1'), 'PACK');
84 return { header: pack.readUInt32BE(8), ...by };
86const argOf = (name, body) => { const m = new RegExp(`(?:^|\\n)[0-9a-f]{4}${name} ([^\\n]*)\\n`).exec(body); return m ? m[1] : null; };
87// each exchange as `METHOD filter deepen` (null: the argument is not in the request)
88const said = log => log.map(x => `${x.method} ${argOf('filter', x.body)} ${argOf('deepen', x.body)}`);
89const commitBytes = () => new Uint8Array(gitBuf(bare, ['cat-file', 'commit', tip]));
91// Cloudflare Artifacts' answer to any filter but blob:none (observed 2026-10-23, fetch.ts): HTTP 400 and this text.
92const onlyBlobNone = ({ method, body }) => {
93 const filter = argOf('filter', body);
94 if (method !== 'POST' || filter === null || filter === 'blob:none') return null;
95 return new Response('Unsupported Git fetch filter; only blob:none is supported.', { status: 400, headers: { 'content-type': 'text/plain; charset=utf-8' } });
96};
97// A server that does not implement shallow fetches: its advertisement's fetch= line has no `shallow`, and its
98// upload-pack dies on the `deepen` argument without a word (an empty body), as git's does on an argument it does not take.
99const noShallow = async ({ url: u, method, body }) => {
100 if (method === 'POST' && /deepen /.test(body)) return new Response(new Uint8Array(0), { status: 200, headers: { 'content-type': 'application/x-git-upload-pack-result' } });
101 if (method !== 'GET') return null;
102 const res = await fetch(u, { headers: { 'git-protocol': 'version=2' } });
103 const text = Buffer.from(await res.arrayBuffer()).toString('latin1');
104 const m = /[0-9a-f]{4}fetch=([^\n]*)\n/.exec(text);
105 assert.ok(m && m[1].split(' ').includes('shallow'), 'git advertises shallow');
106 const line = `fetch=${m[1].split(' ').filter(f => f !== 'shallow').join(' ')}\n`;
107 const out = text.replace(m[0], `${(line.length + 4).toString(16).padStart(4, '0')}${line}`);
108 return new Response(Buffer.from(out, 'latin1'), { status: res.status, headers: res.headers });
109};
111test('the blob:none fallback, learned record: one request with deepen 1, the commit and its one tree', async () => {
112 for (const expectType of [null, TYPE.COMMIT]) {
113 const { t, log } = recording(url, { treeFilter: 'blob:none' });
114 const got = unwrap(await fetchObject(t, { oid: tip, expectType }));
115 assert.equal(got.type, TYPE.COMMIT);
116 assert.deepEqual(got.data, commitBytes());
117 assert.equal(got.wide, true);
118 assert.deepEqual(said(log), ['POST blob:none 1']);
119 assert.deepEqual(counted(log[0].bytes), { header: 2, commit: 1, tree: 1, blob: 0, tag: 0 });
120 }
121});
123test('the blob:none fallback, tree:0 refused in this call: both requests carry deepen 1, the answer is the commit and its one tree', async () => {
124 const { t, log } = recording(url, {}, onlyBlobNone);
125 const got = unwrap(await fetchObject(t, { oid: tip }));
126 assert.equal(got.type, TYPE.COMMIT);
127 assert.deepEqual(got.data, commitBytes());
128 assert.equal(got.wide, true);
129 assert.equal(t.quirks.treeFilter, 'blob:none');
130 assert.deepEqual(said(log), ['POST tree:0 1', 'POST blob:none 1']);
131 assert.deepEqual(log.map(x => x.status), [400, 200]);
132 assert.deepEqual(counted(log[1].bytes), { header: 2, commit: 1, tree: 1, blob: 0, tag: 0 });
133});
135test('the unfiltered fallback, stock git without allowFilter: POST (empty), GET, POST with deepen 1 and no filter: the commit, its tree, its blob', async () => {
136 const { t, log } = recording(stockUrl);
137 const got = unwrap(await fetchObject(t, { oid: tip }));
138 assert.equal(got.type, TYPE.COMMIT);
139 assert.deepEqual(got.data, commitBytes());
140 assert.equal(got.wide, true);
141 assert.equal(t.quirks.treeFilter, 'none');
142 assert.deepEqual(said(log), ['POST tree:0 1', 'GET null null', 'POST null 1']);
143 // git's upload-pack died on `filter`: nothing came back
144 assert.equal(log[0].bytes.length, 0);
145 assert.deepEqual(counted(log[2].bytes), { header: 3, commit: 1, tree: 1, blob: 1, tag: 0 });
146});
148test('the unfiltered fallback, learned record (quirks.treeFilter none): one request with deepen 1 and no filter', async () => {
149 for (const expectType of [null, TYPE.COMMIT]) {
150 const { t, log } = recording(stockUrl, { treeFilter: 'none' });
151 const got = unwrap(await fetchObject(t, { oid: tip, expectType }));
152 assert.equal(got.type, TYPE.COMMIT);
153 assert.deepEqual(got.data, commitBytes());
154 assert.equal(got.wide, true);
155 assert.deepEqual(said(log), ['POST null 1']);
156 assert.deepEqual(counted(log[0].bytes), { header: 3, commit: 1, tree: 1, blob: 1, tag: 0 });
157 }
158});
160test("the caller's filter '' with a commit want: one request with deepen 1 and no filter; a caller's deepen 0 is the history", async () => {
161 const { t, log } = recording(url);
162 const got = unwrap(await fetchObject(t, { oid: tip, filter: '', expectType: TYPE.COMMIT }));
163 assert.deepEqual(got.data, commitBytes());
164 assert.deepEqual(said(log), ['POST null 1']);
165 assert.deepEqual(counted(log[0].bytes), { header: 3, commit: 1, tree: 1, blob: 1, tag: 0 });
166 // the caller's depth wins on this path too
167 const all = recording(url);
168 const whole = unwrap(await fetchObject(all.t, { oid: tip, filter: '', expectType: TYPE.COMMIT, deepen: 0 }));
169 assert.deepEqual(whole.data, commitBytes());
170 assert.deepEqual(said(all.log), ['POST null null']);
171 assert.deepEqual(counted(all.log[0].bytes), { header: 3 * N, commit: N, tree: N, blob: N, tag: 0 });
172});
174test('a server without shallow, not yet known, the wide path: one refused deepen, the advertisement read once, then no deepen', async () => {
175 const { t, log } = recording(url, { treeFilter: 'blob:none' }, noShallow);
176 const got = unwrap(await fetchObject(t, { oid: tip, expectType: TYPE.COMMIT }));
177 assert.deepEqual(got.data, commitBytes());
178 assert.equal(got.wide, true);
179 assert.deepEqual(said(log), ['POST blob:none 1', 'GET null null', 'POST blob:none null']);
180 // as before the default depth: the history, without its blobs
181 assert.deepEqual(counted(log[2].bytes), { header: 2 * N, commit: N, tree: N, blob: 0, tag: 0 });
182 // the record has learned it: the next call asks once, without deepen
183 const again = unwrap(await fetchObject(t, { oid: tip, filter: 'blob:none', expectType: TYPE.COMMIT }));
184 assert.deepEqual(again.data, commitBytes());
185 assert.deepEqual(said(log.slice(3)), ['POST blob:none null']);
186});
188test("a server without shallow, not yet known, the caller's filter '': one refused deepen, the advertisement read once, then no deepen", async () => {
189 const { t, log } = recording(url, {}, noShallow);
190 const got = unwrap(await fetchObject(t, { oid: tip, filter: '', expectType: TYPE.COMMIT }));
191 assert.deepEqual(got.data, commitBytes());
192 assert.deepEqual(said(log), ['POST null 1', 'GET null null', 'POST null null']);
193 assert.deepEqual(counted(log[2].bytes), { header: 3 * N, commit: N, tree: N, blob: N, tag: 0 });
194 const again = unwrap(await fetchObject(t, { oid: tip, filter: '', expectType: TYPE.COMMIT }));
195 assert.deepEqual(again.data, commitBytes());
196 assert.deepEqual(said(log.slice(3)), ['POST null null']);
197});
199test('expectType tag carries the default depth: a tag want is the tag and the one commit it peels to, a commit id is one commit', async () => {
200 // Only the first request and git's answer to it are pinned here: what fetchObject makes of a tag want's two objects
201 // belongs to fetchobject-deepen.test.js and fetchobject-deepen-refusal.test.js.
202 const a = recording(url);
203 await fetchObject(a.t, { oid: tag, expectType: TYPE.TAG });
204 assert.equal(said(a.log)[0], 'POST tree:0 1');
205 // depth 1 of a tag is the commit it peels to (upload-pack get_shallow_commits: deref_tag); it was N + 1 objects
206 assert.deepEqual(counted(a.log[0].bytes), { header: 2, commit: 1, tree: 0, blob: 0, tag: 1 });
207 // a commit id named a tag: one commit of pack, and the refusal it has always had (pack_corrupt)
208 const b = recording(url);
209 const got = await fetchObject(b.t, { oid: tip, expectType: TYPE.TAG });
210 assert.deepEqual(said(b.log), ['POST tree:0 1']);
211 assert.deepEqual(counted(b.log[0].bytes), { header: 1, commit: 1, tree: 0, blob: 0, tag: 0 });
212 assert.equal(got.ok, false);
213 assert.equal(got.error.code, 'pack_corrupt');
214 assert.equal(got.error.message, 'expected tag, got commit');
215});
Diagnostics6 git calls · 93 ms
CallStatusSizeTime093 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=82
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2004.2 KiB82 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=4269
git.response.cache=MISS
git.response.duration_ms=82
git.fetch.wants=e9e69045742098dd1fb1ba6132333c2b4fffb879
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4269
git.pkt.payload.max=4242
git.sideband.band1.bytes=4242
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0fcf70f4e332c
Request
6b94008a…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format