GitHubivarvong

ply

main
ply7 git calls88 ms28 KiB5 cachedprotocol v2
228 lines12 KiBRaw
1// The streamed-body cells the #36 matrix does not reach (#81): a declared size one under, at, and one over the
2// inflated length, on read and edit; a header that fails before any body byte; and the blob:none object cap against
3// a delta whose resolved size is past the base. git index-pack is the oracle for every pack. A dropped connection
4// has no pack; its oracle is the stated code.
5import { test, beforeAll as before, afterAll as after } from 'vitest';
6import assert from 'node:assert/strict';
7import { spawnSync } from 'node:child_process';
8import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
9import { tmpdir } from 'node:os';
10import { join } from 'node:path';
11import { tmpRoot, makeBareRepo, startGitServer, commitFiles, git, initRepo } from '../server.js';
12import { editFile } from '../../src/edit.js';
13import { readFile, openBlob } from '../../src/read.js';
14import { fetchObject as fetchObjectR } from '../../src/fetch.js';
15import { Tracer } from '../../src/otel.js';
16import { withLimits, DEFAULT_LIMITS } from '../../src/limits.js';
17import { buildPack, encodeObjHeader, spliceDelta, zlibDeflate } from '../../src/pack.js';
18import { hashObject, TYPE } from '../../src/objects.js';
19import { Sha1 } from '../../src/sha1.js';
20import { concat, te } from '../../src/bytes.js';
21import { pkt, FLUSH } from '../../src/pktline.js';
22import { unwrap, unwrapAsync } from '../unwrap.js';
24// Throwing adapters over the Result API (0.2.0): the assertions below are about the throw.
25const fetchObject = (...a) => unwrapAsync(fetchObjectR(...a));
26const FIND = 'M';
27const AUTHOR = { name: 'adv', email: 'adv@x' };
28const FRAME = 8;
29const ENV = { ...Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith('GIT_'))), GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' };
31const contentOf = n => { const b = new Uint8Array(n); if (n) b[0] = 0x4d; for (let i = 1; i < n; i++) b[i] = 0x78; return b; };
33async function liePack(content, declared) {
34 const z = await zlibDeflate(content);
35 const body = concat([Uint8Array.of(0x50, 0x41, 0x43, 0x4b, 0, 0, 0, 2, 0, 0, 0, 1), encodeObjHeader(TYPE.BLOB, declared), z]);
36 return concat([body, new Sha1().update(body).digest()]);
39// index-pack --strict in a fresh bare repo. Returns the status.
40const indexPack = bytes => {
41 const dir = mkdtempSync(join(tmpdir(), 'ply-idx-'));
42 try {
43 spawnSync('git', ['init', '-q', '--bare', dir], { env: ENV });
44 const file = join(dir, 'p.pack');
45 writeFileSync(file, bytes);
46 const r = spawnSync('git', ['index-pack', '--strict', file], { cwd: dir, env: ENV, encoding: 'utf8' });
47 return r.status;
48 } finally { rmSync(dir, { recursive: true, force: true }); }
49};
51const catBlob = (pack, oid) => {
52 const dir = mkdtempSync(join(tmpdir(), 'ply-cat-'));
53 try {
54 spawnSync('git', ['init', '-q', '--bare', dir], { env: ENV });
55 // index-pack of a named file writes only the .idx beside it; --stdin installs the pack where cat-file reads it
56 const r = spawnSync('git', ['index-pack', '--strict', '--stdin'], { cwd: dir, env: ENV, input: pack });
57 assert.equal(r.status, 0, String(r.stderr));
58 const cat = spawnSync('git', ['cat-file', 'blob', oid], { cwd: dir, env: ENV });
59 assert.equal(cat.status, 0, String(cat.stderr));
60 return cat.stdout;
61 } finally { rmSync(dir, { recursive: true, force: true }); }
62};
64// Band-1 frames of `pack`, 8 bytes at a time, after the packfile section. The real server answers everything else.
65function intercept(oid, pack) {
66 return async (url, init) => {
67 const req = init?.body instanceof Uint8Array ? Buffer.from(init.body).toString('latin1') : '';
68 if (!req.includes('command=fetch\n') || req.includes('filter ') || !req.includes(`want ${oid}\n`)) return fetch(url, init);
69 const frames = [unwrap(pkt(te.encode('packfile\n')))];
70 for (let i = 0; i < pack.length; i += FRAME) frames.push(unwrap(pkt(concat([Uint8Array.of(1), pack.subarray(i, i + FRAME)]))));
71 frames.push(FLUSH);
72 let i = 0;
73 const stream = new ReadableStream({
74 pull(c) { if (i >= frames.length) { c.close(); return; } c.enqueue(frames[i++]); },
75 }, { highWaterMark: 0 });
76 return new Response(stream, { status: 200, headers: { 'content-type': 'application/x-git-upload-pack-result' } });
77 };
80let root, cleanup, srv;
81before(async () => { ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root); });
82after(async () => { await srv.close(); cleanup(); });
84test('declared size one under, at, and one over the inflated length, on read and edit', { timeout: 120_000 }, async () => {
85 const cells = [];
86 for (const n of [0, 1, 64]) for (const declared of (n === 0 ? [0, 1] : [n - 1, n, n + 1])) for (const path of ['read', 'edit']) cells.push({ n, declared, path });
87 assert.equal(cells.length, 16);
88 const packs = new Map();
89 for (const { n, declared } of cells) {
90 const key = `${n}:${declared}`;
91 if (!packs.has(key)) packs.set(key, await liePack(contentOf(n), declared));
92 }
93 for (const [key, pack] of packs) {
94 const [n, declared] = key.split(':').map(Number);
95 const status = indexPack(pack);
96 assert.equal(status === 0, n === declared, `index-pack ${key} status ${status}`);
97 }
98 for (const { n, declared, path } of cells) {
99 const data = contentOf(n);
100 const name = `lie${n}d${declared}${path}`;
101 const bare = makeBareRepo(root, name);
102 const work = join(root, `w${name}`);
103 mkdirSync(work);
104 initRepo(work);
105 const head = commitFiles(work, bare, { 'a.bin': Buffer.from(data) }, 'lie');
106 const oid = hashObject('blob', data);
107 const url = `${srv.url}/${name}.git`;
108 const fetchImpl = intercept(oid, packs.get(`${n}:${declared}`));
109 const opts = { tracer: new Tracer(), fetchImpl };
110 const where = `${path} n=${n} declared=${declared}`;
111 let err = null, got = null;
112 if (path === 'read') {
113 const res = await unwrapAsync(readFile(url, { path: 'a.bin' }, opts));
114 const parts = [];
115 try { for await (const c of res.stream) parts.push(c); got = concat(parts); } catch (e) { err = e; }
116 } else {
117 try { got = await unwrapAsync(editFile(url, { path: 'a.bin', find: FIND, replace: 'N', author: AUTHOR, message: 'lie' }, opts)); }
118 catch (e) { err = e; }
119 }
120 // each cell has its own bare repo; read the ref the operation left, then put main back only so a later
121 // assertion cannot observe a push from this cell
122 const ref = git(bare, ['rev-parse', 'main']).trim();
123 if (ref !== head) git(bare, ['update-ref', 'refs/heads/main', head]);
124 if (n === declared && path === 'read') {
125 assert.equal(err, null, where);
126 assert.ok(Buffer.from(got).equals(Buffer.from(data)), where);
127 } else if (n === declared && path === 'edit' && n === 0) {
128 assert.equal(err?.code, 'match_count', `${where}: ${err?.message}`);
129 assert.equal(ref, head, where);
130 } else if (n === declared) {
131 assert.equal(err, null, `${where}: ${err?.message}`);
132 assert.notEqual(ref, head, where);
133 } else if (declared < n) {
134 assert.equal(err?.code, 'pack_corrupt', `${where}: ${err?.message}`);
135 assert.equal(err.message, `server streamed more than the ${declared} bytes it declared`, where);
136 assert.equal(ref, head, where);
137 } else {
138 assert.equal(err?.code, 'pack_corrupt', `${where}: ${err?.message}`);
139 assert.equal(err.message, path === 'read' ? `inflated ${n} != declared ${declared}` : `inflated ${n} != header size ${declared}`, where);
140 assert.equal(ref, head, where);
141 }
142 }
143});
145test('a header that fails before the body: a raw reset, and a bad signature git also refuses', async () => {
146 const tracer = new Tracer();
147 const parent = tracer.start('p');
148 const oid = 'a'.repeat(40);
149 const limits = DEFAULT_LIMITS;
150 const transport = frames => ({
151 kind: 'http', url: 'http://fake/r.git', tracer, parent: null, limits, signal: null, quirks: {}, objects: new Map(), scrub: null,
152 frames, advertise: async () => { throw new Error('unused'); },
153 });
154 // openBlob returns the header failure (#145); it does not reject
155 const reset = await openBlob(transport(async function* () { throw new Error('reset'); }), oid, { follows: parent });
156 assert.equal(reset.ok, false);
157 assert.equal(reset.error.code, 'pack_corrupt');
158 assert.equal(reset.error.message, 'blob stream: reset');
159 const bad = te.encode('NOPE not a pack!!!!');
160 assert.notEqual(indexPack(bad), 0);
161 const frames = async function* () {
162 yield { kind: 'data', payload: te.encode('packfile\n') };
163 yield { kind: 'data', payload: concat([Uint8Array.of(1), bad]) };
164 yield { kind: 'flush' };
165 };
166 const sig = await openBlob(transport(frames), oid, { follows: parent });
167 assert.equal(sig.ok, false);
168 assert.equal(sig.error.code, 'pack_corrupt');
169 assert.equal(sig.error.message, 'pack: bad signature');
170});
172test('blob:none object cap: one under, at, and one past each size the pack meets', { timeout: 60_000 }, async () => {
173 // A ref-delta's header size is the delta's, not the resolved object's. parsePack refuses a header over maxObjectSize
174 // ('pack object size') before fetchWide can see the resolved bytes ('object size', and only while maxDeltaResult
175 // stays at its default, above the result). Caps are one under, at, and one past the base header, the delta header,
176 // and the resolved length. git index-pack accepts every pack; cat-file is the byte oracle.
177 const bases = [8, 32], expands = [2, 16];
178 const built = [];
179 for (const baseN of bases) for (const extraN of expands) {
180 const base = new Uint8Array(baseN).fill(0x41);
181 const extra = new Uint8Array(extraN).fill(0x42);
182 const result = concat([base, extra]);
183 const { delta } = spliceDelta({ baseLen: baseN, off: baseN, oldLen: 0, insert: extra });
184 const { pack, entries } = await buildPack([{ type: TYPE.BLOB, data: base }, { type: TYPE.BLOB, data: delta, base: { tag: 'ref', oid: hashObject('blob', base) } }]);
185 const oid = hashObject('blob', result);
186 const where = `base ${baseN} +${extraN}`;
187 assert.ok(Buffer.from(catBlob(pack, oid)).equals(Buffer.from(result)), where);
188 const deltaN = entries[1].size;
189 assert.equal(deltaN, delta.length, where);
190 built.push({ pack, result, oid, baseN, extraN, deltaN, resultN: result.length });
191 }
192 const cells = [];
193 for (const b of built) {
194 const caps = new Set();
195 for (const n of [b.baseN, b.deltaN, b.resultN]) for (const cap of [n - 1, n, n + 1]) caps.add(cap);
196 for (const cap of [...caps].sort((a, c) => a - c)) cells.push({ ...b, cap });
197 }
198 assert.equal(cells.length, 32);
199 const tracer = new Tracer();
200 let resolved = 0;
201 for (const b of cells) {
202 const limits = withLimits({ maxObjectSize: b.cap });
203 const frames = async function* () {
204 yield { kind: 'data', payload: te.encode('packfile\n') };
205 for (let i = 0; i < b.pack.length; i += FRAME) yield { kind: 'data', payload: concat([Uint8Array.of(1), b.pack.subarray(i, i + FRAME)]) };
206 yield { kind: 'flush' };
207 };
208 const t = {
209 kind: 'http', url: 'http://fake/r.git', tracer, parent: null, limits, signal: null, quirks: { treeFilter: 'blob:none' }, objects: new Map(), scrub: null, frames,
210 advertise: async () => { throw new Error('unused') },
211 };
212 const where = `base ${b.baseN} +${b.extraN} delta ${b.deltaN} result ${b.resultN} cap ${b.cap}`;
213 const res = await fetchObject(t, { oid: b.oid }).then(v => ({ v }), e => ({ e }));
214 // the base is the first object, so a cap under it never reaches the delta header
215 const refused = b.cap < b.baseN ? ['pack object size', b.baseN] : b.cap < b.deltaN ? ['pack object size', b.deltaN] : b.cap < b.resultN ? ['object size', b.resultN] : null;
216 if (refused) {
217 assert.equal(res.e?.code, 'limit_exceeded', `${where}: ${res.e?.message}`);
218 assert.equal(res.e.what, refused[0], where);
219 assert.equal(res.e.got, refused[1], where);
220 assert.equal(res.e.max, b.cap, where);
221 } else {
222 assert.equal(res.e, undefined, `${where}: ${res.e?.message}`);
223 assert.ok(Buffer.from(res.v.data).equals(Buffer.from(b.result)), where);
224 }
225 if (res.e?.what === 'object size') resolved++;
226 }
227 assert.ok(resolved > 0, 'a cap between the headers and the resolved length');
228});
Diagnostics7 git calls · 88 ms
CallStatusSizeTime088 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetchtree:02001.4 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=1428
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=74d7bb09987bbe78a9c0ea8f43a1b5a4e079f538
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1428
git.pkt.payload.max=1401
git.sideband.band1.bytes=1401
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1401
git.pack.trailer=65ea1a3aeef333a80a2dfa3f441e629d90497e8e
Open pack map
fetch2004.2 KiB77 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=77
git.response.bytes=4296
git.response.cache=MISS
git.response.duration_ms=77
git.fetch.wants=0f974146bedd6de3eaca9d10b936c6f236c37a83
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4296
git.pkt.payload.max=4269
git.sideband.band1.bytes=4269
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2f1a91d7afda9
Request
ad469737…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format