GitHubivarvong

ply

main
ply6 git calls114 ms27 KiB3 cachedprotocol v2
221 lines13 KiBRaw
1// "Without the capability ply behaves as on main", for every refusal shape. fetchObject adds `deepen 1` to a want that
2// may be a commit. A server whose fetch= line lists no `shallow` may refuse that argument with an ERR pkt-line or with
3// HTTP 400 (not only with an empty body: test/fetchobject-deepen.test.js). ply then reads the advertisement once per
4// quirks record and asks again without `deepen`: the request main sends, so the answer is main's.
5// The oracle is git: N commits in a line served by git http-backend; the proxy in front of it strips `shallow` from the
6// advertisement and refuses any request that carries `deepen`. Expected values are hand-derived from git's rules and
7// counted by git verify-pack: with no depth a commit want is answered with every commit reachable from it, so
8// filter tree:0 N commits (trees and blobs dropped)
9// filter blob:none N commits + N root trees (every commit changes f.txt: N distinct trees of one entry)
10// no filter N commits + N trees + N blobs
11// and the object returned is `git cat-file commit <tip>`.
12import { test, beforeAll as before, afterAll as after } from 'vitest';
13import assert from 'node:assert/strict';
14import { join } from 'node:path';
15import { mkdirSync, writeFileSync } from 'node:fs';
16import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer } from './server.js';
17import { fetchObject } from '../src/fetch.js';
18import { createTransport } from '../src/http.js';
19import { TYPE } from '../src/objects.js';
20import { unwrap } from './unwrap.js';
22const N = 24;
23let root, cleanup, srv, bare, url, tip, k = 0;
25before(async () => {
26 ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root);
27 bare = makeBareRepo(root, 'hist'); url = `${srv.url}/hist.git`;
28 let stream = '';
29 for (let i = 1; i <= N; i++) {
30 const msg = `c${i}\n`, body = `line ${i}\n`;
31 stream += `commit refs/heads/main\nmark :${i}\ncommitter Ada <ada@x> ${1_700_000_000 + i} +0000\ndata ${msg.length}\n${msg}${i > 1 ? `from :${i - 1}\n` : ''}M 100644 inline f.txt\ndata ${body.length}\n${body}\n`;
32 }
33 gitBuf(bare, ['fast-import', '--quiet'], Buffer.from(stream));
34 tip = git(bare, ['rev-parse', 'main']).trim();
35 assert.equal(Number(git(bare, ['rev-list', '--count', 'main']).trim()), N);
36});
37after(async () => { await srv.close(); cleanup(); });
39// A transport whose every exchange is kept; `rewrite` may answer in the server's place.
40function recording(rewrite, quirks = { treeFilter: 'tree:0', blobWants: 'many' }) {
41 const log = [];
42 const fetchImpl = async (input, init = {}) => {
43 const body = init.body === undefined || init.body === null ? '' : Buffer.from(await new Response(init.body).arrayBuffer()).toString('latin1');
44 const method = init.method ?? 'GET';
45 const faked = await rewrite({ url: String(input), method, body });
46 const res = faked ?? await fetch(input, { ...init, body: body === '' ? undefined : Buffer.from(body, 'latin1') });
47 const bytes = Buffer.from(await res.arrayBuffer());
48 log.push({ method, body, bytes, status: res.status });
49 return new Response(bytes, { status: res.status, headers: res.headers });
50 };
51 const t = unwrap(createTransport(url, { fetchImpl, quirks }));
52 return { t, log };
54const deepenOf = body => { const m = /(?:^|\n)[0-9a-f]{4}deepen (\d+)\n/.exec(body); return m ? Number(m[1]) : null; };
55const filterOf = body => { const m = /(?:^|\n)[0-9a-f]{4}filter ([^\n]+)\n/.exec(body); return m ? m[1] : null; };
56const shape = log => log.map(x => `${x.method} ${deepenOf(x.body)}`);
57// The band-1 bytes of a fetch response (the pack), counted by git.
58function counted(bytes) {
59 const parts = [];
60 let i = 0, inPack = false;
61 while (i + 4 <= bytes.length) {
62 const len = parseInt(bytes.subarray(i, i + 4).toString('latin1'), 16);
63 if (len < 4) { i += 4; continue; }
64 const payload = bytes.subarray(i + 4, i + len); i += len;
65 if (inPack) { if (payload[0] === 1) parts.push(payload.subarray(1)); continue; }
66 if (payload.toString('latin1').trimEnd() === 'packfile') inPack = true;
67 }
68 const dir = join(root, `count${k++}`); mkdirSync(dir); git(dir, ['init', '-q', '--bare', '.']);
69 const file = join(dir, 'x.pack'); writeFileSync(file, Buffer.concat(parts));
70 git(dir, ['index-pack', file]);
71 const by = { commit: 0, tree: 0, blob: 0, tag: 0 };
72 for (const l of git(dir, ['verify-pack', '-v', join(dir, 'x.idx')]).split('\n')) { const m = /^[0-9a-f]{40} (commit|tree|blob|tag) /.exec(l); if (m) by[m[1]]++; }
73 return by;
75const raw = () => new Uint8Array(gitBuf(bare, ['cat-file', 'commit', tip]));
77// The advertisement with features taken off its fetch= line: a server that does not implement them.
78const strip = async (u, drop) => {
79 const res = await fetch(u, { headers: { 'git-protocol': 'version=2' } });
80 const text = Buffer.from(await res.arrayBuffer()).toString('latin1');
81 const m = /[0-9a-f]{4}fetch=([^\n]*)\n/.exec(text);
82 for (const f of drop) assert.ok(m && m[1].split(' ').includes(f), `git advertises ${f}`);
83 const line = `fetch=${m[1].split(' ').filter(f => !drop.includes(f)).join(' ')}\n`;
84 return new Response(Buffer.from(text.replace(m[0], `${(line.length + 4).toString(16).padStart(4, '0')}${line}`), 'latin1'), { status: res.status, headers: res.headers });
85};
86// oxlint-disable-next-line typescript/promise-function-async -- strip's promise, passed through
87const stripShallow = u => strip(u, ['shallow']);
88const pkt = s => { const b = Buffer.from(s, 'latin1'); return Buffer.concat([Buffer.from((b.length + 4).toString(16).padStart(4, '0'), 'latin1'), b]); };
89// How a request that carries `deepen` is refused. ERR: a pkt-line, HTTP 200 (git's own way to refuse an argument it
90// parses and rejects). A status: the body a non-git server gives (Artifacts refuses a filter it does not take with 400).
91const REFUSALS = {
92 err: () => new Response(pkt('ERR unknown argument deepen 1\n'), { status: 200, headers: { 'content-type': 'application/x-git-upload-pack-result' } }),
93 400: () => new Response('bad argument', { status: 400 }),
94 401: () => new Response('no', { status: 401 }),
95 403: () => new Response('no', { status: 403 }),
96 404: () => new Response('no', { status: 404 }),
97 429: () => new Response('slow down', { status: 429 }),
98 500: () => new Response('broken', { status: 500 }),
99};
100// `lists`: whether the advertisement still lists `shallow` (a server that takes it and fails for another reason).
101const refusing = (how, lists = false) => async ({ url: u, method, body }) => {
102 if (method === 'POST' && /deepen /.test(body)) return REFUSALS[how]();
103 return method === 'GET' && !lists ? stripShallow(u) : null;
104};
106// The three paths of objectAt, as [name, quirks.treeFilter, the caller's filter (undefined: the default tree:0), the
107// filter on the wire, the pack main's request is answered with].
108const PATHS = [
109 ['tree:0 single', 'tree:0', undefined, 'tree:0', { commit: N, tree: 0, blob: 0, tag: 0 }],
110 ['wide blob:none', 'blob:none', undefined, 'blob:none', { commit: N, tree: N, blob: 0, tag: 0 }],
111 ['wide unfiltered', 'none', undefined, null, { commit: N, tree: N, blob: N, tag: 0 }],
112 ["filter ''", 'tree:0', '', null, { commit: N, tree: N, blob: N, tag: 0 }],
113];
114const want = filter => (filter === undefined ? { oid: tip, expectType: TYPE.COMMIT } : { oid: tip, expectType: TYPE.COMMIT, filter });
116for (const how of ['err', '400']) {
117 for (const [name, treeFilter, filter, wire, pack] of PATHS) {
118 test(`no shallow, deepen refused (${how}), ${name}: POST, GET, POST without deepen, and main's answer`, async () => {
119 const { t, log } = recording(refusing(how), { treeFilter, blobWants: 'many' });
120 const got = await fetchObject(t, want(filter));
121 assert.deepEqual(shape(log), ['POST 1', 'GET null', 'POST null'], got.ok ? 'ok' : `${got.error.code}: ${got.error.message}`);
122 // the second request is the first without its depth: what main sends this server
123 assert.equal(log[2].body, log[0].body.replace('000ddeepen 1\n', ''));
124 assert.equal(filterOf(log[2].body), wire);
125 assert.equal(got.ok, true);
126 assert.equal(got.value.type, TYPE.COMMIT);
127 assert.deepEqual(got.value.data, raw());
128 assert.deepEqual(counted(log[2].bytes), pack);
129 });
130 }
133test('no shallow, once learned: the next call on the same quirks record sends no deepen and reads nothing', async () => {
134 for (const how of ['err', '400']) {
135 for (const filter of [undefined, '']) {
136 const { t, log } = recording(refusing(how));
137 unwrap(await fetchObject(t, want(filter)));
138 assert.deepEqual(shape(log), ['POST 1', 'GET null', 'POST null']);
139 const again = unwrap(await fetchObject(t, want(filter)));
140 assert.deepEqual(again.data, raw());
141 assert.deepEqual(shape(log).slice(3), ['POST null']);
142 }
143 }
144});
146test('never asked again: 401, 403, 404, 429 and 500 stand as they are, with no advertisement read', async () => {
147 const codes = { 401: 'auth', 403: 'auth', 404: 'not_found', 429: 'rate_limited', 500: 'upstream_5xx' };
148 for (const [how, code] of Object.entries(codes)) {
149 for (const [name, treeFilter, filter] of PATHS) {
150 const { t, log } = recording(refusing(how), { treeFilter, blobWants: 'many' });
151 const got = await fetchObject(t, want(filter));
152 assert.equal(got.ok, false, `${how} ${name}`);
153 assert.equal(got.error.code, code, `${how} ${name}`);
154 assert.deepEqual(shape(log), ['POST 1'], `${how} ${name}`);
155 }
156 }
157});
159test("never asked again: the caller's own deepen is sent as given and its refusal stands", async () => {
160 for (const [how, code] of [['err', 'remote_error'], ['400', 'http']]) {
161 for (const [name, treeFilter, filter] of PATHS) {
162 const { t, log } = recording(refusing(how), { treeFilter, blobWants: 'many' });
163 const got = await fetchObject(t, { ...want(filter), deepen: 1 });
164 assert.equal(got.ok, false, `${how} ${name}`);
165 assert.equal(got.error.code, code, `${how} ${name}`);
166 assert.deepEqual(shape(log), ['POST 1'], `${how} ${name}`);
167 }
168 }
169});
171test('a server that lists shallow: the first error stands, and the advertisement is read once per quirks record', async () => {
172 for (const [how, code, said] of [['err', 'remote_error', /unknown argument deepen 1/], ['400', 'http', /^HTTP 400 /]]) {
173 for (const [name, treeFilter, filter] of PATHS) {
174 const { t, log } = recording(refusing(how, true), { treeFilter, blobWants: 'many' });
175 const got = await fetchObject(t, want(filter));
176 assert.equal(got.ok, false, `${how} ${name}`);
177 assert.equal(got.error.code, code, `${how} ${name}`);
178 assert.match(got.error.message, said);
179 assert.deepEqual(shape(log), ['POST 1', 'GET null'], `${how} ${name}`);
180 // shallow is now known for the record: the same refusal is not a reason to read the advertisement again
181 const again = await fetchObject(t, want(filter));
182 assert.equal(again.ok, false);
183 assert.equal(again.error.code, code);
184 assert.deepEqual(shape(log).slice(2), ['POST 1'], `${how} ${name}`);
185 }
186 }
187});
189// The request without `deepen` is main's request, so main's handling of its answer is ply's (R3 on the tree:0 path).
190// Two servers without `shallow` that main reads through its filter fallbacks, each refusing the default depth first in
191// one of the three shapes. The expected values are main's outcome, derived from git's rules as above:
192// (a) lists `filter`, takes only blob:none (tree:0: HTTP 400, Artifacts' text). Main: tree:0 -> 400, blob:none -> the
193// commit, in a pack of N commits + N root trees.
194// (b) lists neither `shallow` nor `filter` and dies on a `filter` line as git's upload-pack does (an empty body).
195// Main: tree:0 -> empty, the advertisement, no filter -> the commit, in a pack of N commits + N trees + N blobs.
196const EMPTY = () => new Response(new Uint8Array(0), { status: 200, headers: { 'content-type': 'application/x-git-upload-pack-result' } });
197const refusingFirst = how => (how === 'empty' ? EMPTY() : REFUSALS[how]());
198const SERVERS = [
199 ['lists filter, takes only blob:none', ['shallow'], () => new Response('Unsupported Git fetch filter; only blob:none is supported.', { status: 400, headers: { 'content-type': 'text/plain; charset=utf-8' } }), body => filterOf(body) === 'tree:0',
200 ['POST 1 tree:0', 'GET null null', 'POST null tree:0', 'POST null blob:none'], 'blob:none', { commit: N, tree: N, blob: 0, tag: 0 }],
201 ['lists no filter, dies on one', ['shallow', 'filter'], EMPTY, body => filterOf(body) !== null,
202 ['POST 1 tree:0', 'GET null null', 'POST null null'], 'none', { commit: N, tree: N, blob: N, tag: 0 }],
203];
204for (const how of ['err', '400', 'empty']) {
205 for (const [name, drop, refuseFilter, refused, requests, learned, pack] of SERVERS) {
206 test(`no shallow, deepen refused (${how}), tree:0 path, server ${name}: main's answer by main's fallback`, async () => {
207 const { t, log } = recording(async ({ url: u, method, body }) => {
208 if (method === 'GET') return strip(u, drop);
209 if (/deepen /.test(body)) return refusingFirst(how);
210 return refused(body) ? refuseFilter() : null;
211 });
212 const got = await fetchObject(t, want(undefined));
213 assert.equal(got.ok, true, got.ok ? 'ok' : `${got.error.code}: ${got.error.message} after ${JSON.stringify(log.map(x => `${x.method} ${deepenOf(x.body)} ${filterOf(x.body)} -> ${x.status} ${x.bytes.length}B`))}`);
214 assert.equal(got.value.type, TYPE.COMMIT);
215 assert.deepEqual(got.value.data, raw());
216 assert.deepEqual(log.map(x => `${x.method} ${deepenOf(x.body)} ${filterOf(x.body)}`), requests);
217 assert.deepEqual(counted(log.at(-1).bytes), pack);
218 assert.equal(t.quirks.treeFilter, learned);
219 });
220 }
Diagnostics6 git calls · 114 ms
CallStatusSizeTime0114 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=103
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2004.5 KiB103 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=4576
git.response.cache=MISS
git.response.duration_ms=103
git.fetch.wants=4e37e481205af2b812730355fca228445f58c929
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4576
git.pkt.payload.max=4549
git.sideband.band1.bytes=4549
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e157ae285390ce
Request
67752a8d…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format