GitHubivarvong

ply

main
ply7 git calls33 ms28 KiB5 cachedprotocol v2
164 lines9.8 KiBRaw
1// The default-depth space (test/space/default-depth.space.js): every cell of asked × expectType × knowledge ×
2// advertised × refusal × path through fetchObject against git http-backend behind a proxy that renders the cell's
3// server. The proxy keeps git's answer to each distinct request (the product has 1,800 cells and a few dozen distinct
4// requests), so every pack and every advertisement is git's own bytes. Checked per cell: the requests in order with
5// their `deepen` and `filter`, the outcome, the object against `git cat-file`, and the pack's object count (its own
6// header) against the hand-derived count. SPACE.md's block is generated here (DEFAULT_DEPTH_WRITE=1 rewrites it).
7import { test, beforeAll as before, afterAll as after } from 'vitest';
8import assert from 'node:assert/strict';
9import { readFileSync, writeFileSync } from 'node:fs';
10import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer } from '../server.js';
11import { fetchObject } from '../../src/fetch.js';
12import { discover } from '../../src/discover.js';
13import { createTransport } from '../../src/http.js';
14import { TYPE } from '../../src/objects.js';
15import { unwrap } from '../unwrap.js';
16import { cells, expected, block, START, END } from './default-depth.space.js';
18const N = 6;
19const SPACE_MD = new URL('../../SPACE.md', import.meta.url);
20let root, cleanup, srv, bare, url, wants;
22before(async () => {
23 ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root);
24 bare = makeBareRepo(root, 'hist'); url = `${srv.url}/hist.git`;
25 let stream = '';
26 for (let i = 1; i <= N; i++) {
27 const msg = `c${i}\n`, body = `line ${i}\n`;
28 stream += `commit refs/heads/main\nmark :${i}\ncommitter Ada <ada@x> ${1_700_000_000 + i} +0000\ndata ${msg.length}\n${msg}${i > 1 ? `from :${i - 1}\n` : ''}M 100644 inline f.txt\ndata ${body.length}\n${body}\n`;
29 }
30 gitBuf(bare, ['fast-import', '--quiet'], Buffer.from(stream));
31 git(bare, ['-c', 'user.name=Ada', '-c', 'user.email=ada@x', 'tag', '-a', '-m', 'v1', 'v1', 'main']);
32 const rev = name => git(bare, ['rev-parse', name]).trim();
33 const commit = rev('main');
34 wants = { none: ['commit', commit], COMMIT: ['commit', commit], TAG: ['tag', rev('refs/tags/v1')], TREE: ['tree', rev('main^{tree}')], BLOB: ['blob', rev('main:f.txt')] };
35 // what the hand-derived pack counts rest on: N commits in a line, N distinct root trees, N distinct blobs
36 assert.equal(Number(git(bare, ['rev-list', '--count', 'main']).trim()), N);
37 assert.equal(new Set(git(bare, ['log', '--format=%T', 'main']).trim().split('\n')).size, N);
38 assert.equal(git(bare, ['cat-file', '--batch-all-objects', '--batch-check=%(objecttype)']).split('\n').filter(x => x === 'blob').length, N);
39});
40after(async () => { await srv.close(); cleanup(); });
42const deepenOf = body => { const m = /(?:^|\n)[0-9a-f]{4}deepen (\d+)\n/.exec(body); return m ? Number(m[1]) : null; };
43const filterOf = body => { const m = /(?:^|\n)[0-9a-f]{4}filter ([^\n]+)\n/.exec(body); return m ? m[1] : null; };
44const pkt = s => { const b = Buffer.from(s, 'latin1'); return Buffer.concat([Buffer.from((b.length + 4).toString(16).padStart(4, '0'), 'latin1'), b]); };
45const RESULT = { 'content-type': 'application/x-git-upload-pack-result' };
46const REFUSALS = {
47 empty: () => new Response(new Uint8Array(0), { status: 200, headers: RESULT }),
48 err: () => new Response(pkt('ERR unknown argument deepen\n'), { status: 200, headers: RESULT }),
49 400: () => new Response('bad argument', { status: 400 }),
50};
51// git's answer to each distinct request, kept
52const kept = new Map();
53async function upstream(input, init, body) {
54 const key = `${init.method ?? 'GET'} ${input}\n${body}`;
55 if (!kept.has(key)) {
56 const res = await fetch(input, { ...init, body: body === '' ? undefined : Buffer.from(body, 'latin1') });
57 kept.set(key, { status: res.status, headers: [...res.headers], bytes: Buffer.from(await res.arrayBuffer()) });
58 }
59 return kept.get(key);
61// The advertisement with the features the cell's server does not have taken off its fetch= line.
62function advertisement(real, drop) {
63 const text = real.bytes.toString('latin1');
64 const m = /[0-9a-f]{4}fetch=([^\n]*)\n/.exec(text);
65 assert.ok(m && m[1].split(' ').includes('shallow') && m[1].split(' ').includes('filter'), 'git advertises shallow and filter');
66 const line = `fetch=${m[1].split(' ').filter(f => !drop.includes(f)).join(' ')}\n`;
67 return Buffer.from(text.replace(m[0], `${(line.length + 4).toString(16).padStart(4, '0')}${line}`), 'latin1');
69// The object count in the header of the pack a fetch response carries (band 1 after `packfile`).
70function packCount(bytes) {
71 const parts = [];
72 let i = 0, inPack = false;
73 while (i + 4 <= bytes.length) {
74 const len = parseInt(bytes.subarray(i, i + 4).toString('latin1'), 16);
75 if (len < 4) { i += 4; continue; }
76 const payload = bytes.subarray(i + 4, i + len); i += len;
77 if (inPack) { if (payload[0] === 1) parts.push(payload.subarray(1)); continue; }
78 if (payload.toString('latin1').trimEnd() === 'packfile') inPack = true;
79 }
80 const pack = Buffer.concat(parts);
81 assert.equal(pack.subarray(0, 4).toString('latin1'), 'PACK');
82 return pack.readUInt32BE(8);
85// The cell's server and a fresh transport (its own quirks record) on it.
86function server(c) {
87 const state = { lists: c.knowledge === 'listed', log: [] };
88 const unfiltered = c.path === 'unfiltered';
89 const fetchImpl = async (input, init = {}) => {
90 const body = init.body === undefined || init.body === null ? '' : Buffer.from(await new Response(init.body).arrayBuffer()).toString('latin1');
91 const method = init.method ?? 'GET';
92 let res;
93 if (method === 'GET') {
94 const real = await upstream(String(input), init, '');
95 const drop = [...(state.lists ? [] : ['shallow']), ...(unfiltered ? ['filter'] : [])];
96 res = { status: real.status, headers: real.headers, bytes: advertisement(real, drop) };
97 } else if (deepenOf(body) !== null && c.refusal in REFUSALS) {
98 const r = REFUSALS[c.refusal]();
99 res = { status: r.status, headers: [...r.headers], bytes: Buffer.from(await r.arrayBuffer()) };
100 } else {
101 res = await upstream(String(input), init, c.refusal === 'ignored' ? body.replace(/000ddeepen \d\n/, '') : body);
102 }
103 state.log.push({ method, body, bytes: res.bytes });
104 return new Response(res.bytes, { status: res.status, headers: res.headers });
105 };
106 const treeFilter = unfiltered ? 'none' : (c.path === 'blob:none' ? 'blob:none' : 'tree:0');
107 const t = unwrap(createTransport(url, { fetchImpl, quirks: { treeFilter, blobWants: 'many' } }));
108 return { t, state };
111test('default depth: every cell sends the expected requests and ends as the rules say, against git', async () => {
112 const all = cells();
113 assert.equal(all.length, 3 * 5 * 3 * 2 * 5 * 4);
114 const wrong = [];
115 const seen = { ok: 0, error: 0 };
116 for (const c of all) {
117 const { t, state } = server(c);
118 if (c.knowledge !== 'unlearned') {
119 const info = unwrap(await discover(t));
120 assert.equal(info.shallow, c.knowledge === 'listed', c.id);
121 }
122 assert.equal(t.quirks.treeFilter, c.path === 'unfiltered' ? 'none' : (c.path === 'blob:none' ? 'blob:none' : 'tree:0'), `${c.id}: the path`);
123 state.lists = c.advertised === 'yes';
124 state.log.length = 0;
125 const [typeName, oid] = wants[c.expectType];
126 const want = { oid };
127 if (c.expectType !== 'none') want.expectType = TYPE[c.expectType];
128 if (c.asked !== 'none') want.deepen = Number(c.asked);
129 if (c.path === "filter ''") want.filter = '';
130 const got = await fetchObject(t, want);
131 const exp = expected(c, N);
132 const requests = state.log.map(x => (x.method === 'GET' ? 'GET' : `POST ${deepenOf(x.body) ?? '-'}`));
133 const outcome = got.ok ? 'ok' : got.error.code;
134 const problems = [];
135 if (requests.join(', ') !== exp.requests.join(', ')) problems.push(`requests ${requests.join(', ')}, expected ${exp.requests.join(', ')}`);
136 if (outcome !== exp.outcome) problems.push(`outcome ${outcome}${got.ok ? '' : ` (${got.error.message})`}, expected ${exp.outcome}`);
137 const wire = c.path === 'tree:0' || c.path === 'blob:none' ? c.path : null;
138 for (const x of state.log) if (x.method === 'POST' && filterOf(x.body) !== wire) problems.push(`filter ${filterOf(x.body)} on the wire, expected ${wire}`);
139 if (exp.objects !== undefined && requests.length === exp.requests.length) {
140 const count = packCount(state.log[state.log.length - 1].bytes);
141 if (count !== exp.objects) problems.push(`pack of ${count} objects, expected ${exp.objects}`);
142 }
143 if (got.ok) {
144 seen.ok++;
145 if (got.value.type !== TYPE[typeName.toUpperCase()]) problems.push(`type ${got.value.type}`);
146 if (!Buffer.from(got.value.data).equals(gitBuf(bare, ['cat-file', typeName, oid]))) problems.push('bytes differ from git cat-file');
147 // nothing returned says the repository is shallow
148 if ('shallow' in got.value) problems.push('the result has a shallow field');
149 } else seen.error++;
150 if (problems.length > 0) wrong.push(`${c.id}: ${problems.join('; ')}`);
151 }
152 assert.deepEqual(wrong.slice(0, 60), [], `${wrong.length} of ${all.length} cells wrong`);
153 // both classes are populated: the walk is not vacuous
154 assert.ok(seen.ok > 600 && seen.error > 300, JSON.stringify(seen));
155}, 240_000);
157test('SPACE.md\'s default-depth block is current', () => {
158 const made = block(N);
159 const md = readFileSync(SPACE_MD, 'utf8');
160 const i = md.indexOf(START), j = md.indexOf(END);
161 if (process.env.DEFAULT_DEPTH_WRITE === '1' && i >= 0 && j > i) { writeFileSync(SPACE_MD, md.slice(0, i) + made + md.slice(j + END.length)); return; }
162 assert.ok(i >= 0 && j > i, 'SPACE.md has no default-depth block: add the markers and run DEFAULT_DEPTH_WRITE=1 npx vitest run test/space/default-depth.test.js');
163 assert.equal(md.slice(i, j + END.length), made, 'SPACE.md default-depth block is stale: run DEFAULT_DEPTH_WRITE=1 npx vitest run test/space/default-depth.test.js');
164});
Diagnostics7 git calls · 33 ms
CallStatusSizeTime033 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=11990
git.response.cache=UPDATING
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetchtree:02001.4 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=1428
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=74d7bb09987bbe78a9c0ea8f43a1b5a4e079f538
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1428
git.pkt.payload.max=1401
git.sideband.band1.bytes=1401
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1401
git.pack.trailer=65ea1a3aeef333a80a2dfa3f441e629d90497e8e
Open pack map
fetch2003.9 KiB · hit20 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=20
git.response.bytes=3964
git.response.cache=HIT
git.response.duration_ms=20
git.fetch.wants=038ffcd47decfa64a2fed1a376713f915149f286
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3964
git.pkt.payload.max=3937
git.sideband.band1.bytes=3937
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e147c56865a140
Request
f6dd7650…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format