main
1// fetchObject costs one object's worth of pack, whatever the history behind the want. The oracle is git: a repository of
2// N commits built by git fast-import, served by git http-backend; every response is recorded, and the pack in it is
3// counted twice, from its own 12-byte header and by git index-pack + git verify-pack. Expected counts are hand-derived:
4// a commit want is 1 commit (it was N: every ancestor, the filter drops trees and blobs, not commits); a tree or blob
5// want is that 1 object; an annotated tag want is what git's server sends for it, stated below.
6import { test, beforeAll as before, afterAll as after } from 'vitest';
7import assert from 'node:assert/strict';
8import { join } from 'node:path';
9import { mkdirSync, writeFileSync } from 'node:fs';
10import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer } from './server.js';
11import { fetchObject } from '../src/fetch.js';
12import { createTransport } from '../src/http.js';
13import { discover } from '../src/discover.js';
14import { TYPE } from '../src/objects.js';
15import { DEFAULT_LIMITS } from '../src/limits.js';
16import { unwrap } from './unwrap.js';
18const N = 300;
19let root, cleanup, srv, bare, url, tip, tree, blob, tag, k = 0;
21before(async () => {
22 ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root);
23 bare = makeBareRepo(root, 'hist'); url = `${srv.url}/hist.git`;
24 // N commits in a line, f.txt changed in each
25 let stream = '';
26 for (let i = 1; i <= N; i++) {
27 const msg = `c${i}\n`, body = `line ${i}\n`;
28 stream += `commit refs/heads/main\nmark :${i}\ncommitter Ada <ada@x> ${1_700_000_000 + i} +0000\ndata ${msg.length}\n${msg}${i > 1 ? `from :${i - 1}\n` : ''}M 100644 inline f.txt\ndata ${body.length}\n${body}\n`;
29 }
30 gitBuf(bare, ['fast-import', '--quiet'], Buffer.from(stream));
31 git(bare, ['-c', 'user.name=Ada', '-c', 'user.email=ada@x', 'tag', '-a', '-m', 'v1', 'v1', 'main']);
32 tip = git(bare, ['rev-parse', 'main']).trim();
33 tree = git(bare, ['rev-parse', 'main^{tree}']).trim();
34 blob = git(bare, ['rev-parse', 'main:f.txt']).trim();
35 tag = git(bare, ['rev-parse', 'refs/tags/v1']).trim();
36 assert.equal(Number(git(bare, ['rev-list', '--count', 'main']).trim()), N);
37 assert.equal(git(bare, ['cat-file', '-t', tag]).trim(), 'tag');
38});
39after(async () => { await srv.close(); cleanup(); });
41// A transport whose every exchange is kept: the request body as text, the response bytes. `rewrite` may answer in the
42// server's place (a server that does not take `deepen`).
43function recording(options = {}, rewrite = null) {
44 const log = [];
45 const fetchImpl = async (input, init = {}) => {
46 const body = init.body === undefined || init.body === null ? '' : Buffer.from(await new Response(init.body).arrayBuffer()).toString('latin1');
47 const method = init.method ?? 'GET';
48 const faked = rewrite === null ? null : await rewrite({ url: String(input), method, body });
49 const res = faked ?? await fetch(input, { ...init, body: body === '' ? undefined : Buffer.from(body, 'latin1') });
50 const bytes = Buffer.from(await res.arrayBuffer());
51 log.push({ url: String(input), method, body, bytes, status: res.status });
52 return new Response(bytes, { status: res.status, headers: res.headers });
53 };
54 const t = unwrap(createTransport(url, { fetchImpl, quirks: { treeFilter: 'tree:0', blobWants: 'many' }, ...options }));
55 return { t, log, fetches: () => log.filter(x => x.method === 'POST' && x.body.includes('command=fetch')) };
56}
57// The response's pkt-lines: the section names before the pack, the lines of shallow-info, the band-1 bytes (the pack).
58function answer(bytes) {
59 const lines = [], parts = [];
60 let i = 0, inPack = false;
61 while (i + 4 <= bytes.length) {
62 const len = parseInt(bytes.subarray(i, i + 4).toString('latin1'), 16);
63 if (len < 4) { i += 4; continue; }
64 const payload = bytes.subarray(i + 4, i + len); i += len;
65 if (inPack) { if (payload[0] === 1) parts.push(payload.subarray(1)); continue; }
66 const text = payload.toString('latin1').trimEnd();
67 if (text === 'packfile') inPack = true; else lines.push(text);
68 }
69 return { lines, pack: Buffer.concat(parts) };
70}
71// The pack counted: by its header, and by git (index-pack accepts it, verify-pack lists its objects).
72function counted(pack) {
73 const dir = join(root, `count${k++}`); mkdirSync(dir); git(dir, ['init', '-q', '--bare', '.']);
74 const file = join(dir, 'x.pack'); writeFileSync(file, pack);
75 git(dir, ['index-pack', file]);
76 const by = { commit: 0, tree: 0, blob: 0, tag: 0 };
77 for (const l of git(dir, ['verify-pack', '-v', join(dir, 'x.idx')]).split('\n')) { const m = /^[0-9a-f]{40} (commit|tree|blob|tag) /.exec(l); if (m) by[m[1]]++; }
78 assert.equal(pack.subarray(0, 4).toString('latin1'), 'PACK');
79 return { header: pack.readUInt32BE(8), ...by };
80}
81const deepenOf = body => { const m = /(?:^|\n)[0-9a-f]{4}deepen (\d+)\n/.exec(body); return m ? Number(m[1]) : null; };
82const raw = (type, oid) => new Uint8Array(gitBuf(bare, ['cat-file', type, oid]));
84test('a commit want is one commit of pack, not the history: expectType commit', async () => {
85 const { t, fetches } = recording();
86 const got = unwrap(await fetchObject(t, { oid: tip, expectType: TYPE.COMMIT }));
87 assert.deepEqual(got.data, raw('commit', tip));
88 assert.equal(fetches().length, 1);
89 assert.equal(deepenOf(fetches()[0].body), 1);
90 const a = answer(fetches()[0].bytes);
91 assert.deepEqual(counted(a.pack), { header: 1, commit: 1, tree: 0, blob: 0, tag: 0 });
92 // the protocol's shallow-info section says where history was cut; nothing ply returns says the repository is shallow
93 assert.deepEqual(a.lines, ['shallow-info', `shallow ${tip}`]);
94 assert.deepEqual(Object.keys(got).toSorted(), ['bust', 'data', 'oid', 'size', 'tree', 'type']);
95 assert.equal(t.objects.size, 0);
96});
98test('a commit want of unknown type (no expectType) is one commit of pack', async () => {
99 const { t, fetches } = recording();
100 const got = unwrap(await fetchObject(t, { oid: tip }));
101 assert.equal(got.type, TYPE.COMMIT);
102 assert.deepEqual(got.data, raw('commit', tip));
103 assert.equal(deepenOf(fetches()[0].body), 1);
104 assert.deepEqual(counted(answer(fetches()[0].bytes).pack), { header: 1, commit: 1, tree: 0, blob: 0, tag: 0 });
105});
107test('a tree want and a blob want of unknown type: git sends the one object and an empty shallow-info', async () => {
108 for (const [oid, type, id] of [[tree, 'tree', TYPE.TREE], [blob, 'blob', TYPE.BLOB]]) {
109 const { t, fetches } = recording();
110 const got = unwrap(await fetchObject(t, { oid }));
111 assert.equal(got.type, id);
112 assert.deepEqual(got.data, raw(type, oid));
113 assert.equal(deepenOf(fetches()[0].body), 1);
114 const a = answer(fetches()[0].bytes);
115 // git's upload-pack opens shallow-info whenever a depth was asked (send_shallow_info) and names no commit: a tree
116 // or blob is not a commit and peels to none (get_shallow_commits skips it)
117 assert.deepEqual(a.lines, ['shallow-info']);
118 assert.deepEqual(counted(a.pack), { header: 1, commit: 0, tree: 0, blob: 0, tag: 0, [type]: 1 });
119 }
120});
122test('a want whose type the caller names tree or blob cannot be a commit: no deepen is sent, as before', async () => {
123 for (const [oid, type, id] of [[tree, 'tree', TYPE.TREE], [blob, 'blob', TYPE.BLOB]]) {
124 const { t, fetches } = recording();
125 const got = unwrap(await fetchObject(t, { oid, expectType: id }));
126 assert.deepEqual(got.data, raw(type, oid));
127 assert.equal(deepenOf(fetches()[0].body), null);
128 const a = answer(fetches()[0].bytes);
129 assert.deepEqual(a.lines, []);
130 assert.deepEqual(counted(a.pack), { header: 1, commit: 0, tree: 0, blob: 0, tag: 0, [type]: 1 });
131 }
132});
134test('an annotated tag want: git sends the tag and the one commit it peels to, not the history', async () => {
135 const { t, fetches } = recording();
136 const got = await fetchObject(t, { oid: tag });
137 assert.equal(deepenOf(fetches()[0].body), 1);
138 const a = answer(fetches()[0].bytes);
139 // depth 1 of a tag is the commit it peels to (get_shallow_commits: deref_tag); that commit is the shallow boundary
140 assert.deepEqual(a.lines, ['shallow-info', `shallow ${tip}`]);
141 assert.deepEqual(counted(a.pack), { header: 2, commit: 1, tree: 0, blob: 0, tag: 1 });
142 // two objects are not a single-object answer: the refusal fetchObject has always given a tag want, now for a pack of
143 // 2 objects where it was N + 1
144 assert.equal(got.ok, false);
145 assert.equal(got.error.code, 'pack_corrupt');
146 assert.equal(got.error.message, 'pack: expected 1 object, got 2');
147});
149test("a caller's deepen is sent as given: 3 is three commits, 0 is the whole history", async () => {
150 const three = recording();
151 const got = unwrap(await fetchObject(three.t, { oid: tip, expectType: TYPE.COMMIT, deepen: 3 }));
152 assert.deepEqual(got.data, raw('commit', tip));
153 assert.equal(deepenOf(three.fetches()[0].body), 3);
154 assert.deepEqual(counted(answer(three.fetches()[0].bytes).pack), { header: 3, commit: 3, tree: 0, blob: 0, tag: 0 });
155 const all = recording();
156 const whole = unwrap(await fetchObject(all.t, { oid: tip, expectType: TYPE.COMMIT, deepen: 0 }));
157 assert.deepEqual(whole.data, raw('commit', tip));
158 assert.equal(deepenOf(all.fetches()[0].body), null);
159 const a = answer(all.fetches()[0].bytes);
160 assert.deepEqual(a.lines, []);
161 assert.deepEqual(counted(a.pack), { header: N, commit: N, tree: 0, blob: 0, tag: 0 });
162});
164test('a history no pack limit would hold in one pack still returns the one commit', async () => {
165 // N commits charge N * (bytes + 3 KiB) against maxInflatedTotal: about 1 MiB here, against a 64 KiB limit
166 const limits = { ...DEFAULT_LIMITS, maxInflatedTotal: 64 * 1024 };
167 const { t, fetches } = recording({ limits });
168 const got = unwrap(await fetchObject(t, { oid: tip, expectType: TYPE.COMMIT }));
169 assert.deepEqual(got.data, raw('commit', tip));
170 assert.deepEqual(counted(answer(fetches()[0].bytes).pack), { header: 1, commit: 1, tree: 0, blob: 0, tag: 0 });
171 // the caller who asks for the whole history under that limit is refused by it, as before
172 const whole = await fetchObject(recording({ limits }).t, { oid: tip, expectType: TYPE.COMMIT, deepen: 0 });
173 assert.equal(whole.ok, false);
174 assert.equal(whole.error.code, 'limit_exceeded');
175 assert.match(whole.error.message, /^pack inflated bytes \d+ exceeds limit 65536$/);
176});
178// A server that does not implement shallow fetches: its advertisement's fetch= line has no `shallow`, and its
179// upload-pack dies on the `deepen` argument without a word (an empty body), as git's does on an argument it does not take.
180const noShallow = async ({ url: u, method, body }) => {
181 if (method === 'POST' && /deepen /.test(body)) return new Response(new Uint8Array(0), { status: 200, headers: { 'content-type': 'application/x-git-upload-pack-result' } });
182 if (method !== 'GET') return null;
183 const res = await fetch(u, { headers: { 'git-protocol': 'version=2' } });
184 const text = Buffer.from(await res.arrayBuffer()).toString('latin1');
185 const m = /[0-9a-f]{4}fetch=([^\n]*)\n/.exec(text);
186 assert.ok(m && m[1].split(' ').includes('shallow'), 'git advertises shallow');
187 const line = `fetch=${m[1].split(' ').filter(f => f !== 'shallow').join(' ')}\n`;
188 const out = text.replace(m[0], `${(line.length + 4).toString(16).padStart(4, '0')}${line}`);
189 return new Response(Buffer.from(out, 'latin1'), { status: res.status, headers: res.headers });
190};
192test('a server that does not advertise shallow: the advertisement in hand, no deepen is sent and the answer is the history', async () => {
193 const { t, fetches } = recording({}, noShallow);
194 const info = unwrap(await discover(t));
195 assert.equal(info.shallow, false);
196 const got = unwrap(await fetchObject(t, { oid: tip, expectType: TYPE.COMMIT }));
197 assert.deepEqual(got.data, raw('commit', tip));
198 assert.equal(fetches().length, 1);
199 assert.equal(deepenOf(fetches()[0].body), null);
200 assert.deepEqual(counted(answer(fetches()[0].bytes).pack), { header: N, commit: N, tree: 0, blob: 0, tag: 0 });
201 // and the limit refuses the history honestly
202 const small = recording({ limits: { ...DEFAULT_LIMITS, maxInflatedTotal: 64 * 1024 } }, noShallow);
203 unwrap(await discover(small.t));
204 const refused = await fetchObject(small.t, { oid: tip, expectType: TYPE.COMMIT });
205 assert.equal(refused.ok, false);
206 assert.equal(refused.error.code, 'limit_exceeded');
207});
209test('a server that does not advertise shallow, not yet known: one refused deepen, the advertisement read once, then no deepen', async () => {
210 const { t, log, fetches } = recording({}, noShallow);
211 const got = unwrap(await fetchObject(t, { oid: tip, expectType: TYPE.COMMIT }));
212 assert.deepEqual(got.data, raw('commit', tip));
213 assert.deepEqual(log.map(x => `${x.method} ${deepenOf(x.body)}`), ['POST 1', 'GET null', 'POST null']);
214 assert.deepEqual(counted(answer(fetches()[1].bytes).pack), { header: N, commit: N, tree: 0, blob: 0, tag: 0 });
215 const again = unwrap(await fetchObject(t, { oid: tip }).then(r => r.ok ? r : fetchObject(t, { oid: tip, expectType: TYPE.COMMIT })));
216 assert.deepEqual(again.data, raw('commit', tip));
217 assert.deepEqual(log.slice(3).map(x => `${x.method} ${deepenOf(x.body)}`).filter(x => x !== 'POST null'), []);
218});
Diagnostics4 git calls · 696 ms
CallStatusSizeTime0696 ms
discover200152 B20 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=20 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B310 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=290 git.response.bytes=145 git.response.cache=EXPIRED git.response.duration_ms=290 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2004.0 KiB382 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=382 git.response.bytes=4145 git.response.cache=MISS git.response.duration_ms=382 git.fetch.wants=5feee6be958bcaa744af3744d9c20cfb03946a50 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4145 git.pkt.payload.max=4124 git.sideband.band1.bytes=4123 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e9e36e9990f3c9
- Request
- a68e3d77…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format