main
1// The streamed-body cells the #36 matrix does not reach (#81): a declared size one under, at, and one over the
2// inflated length, on read and edit; a header that fails before any body byte; and the blob:none object cap against
3// a delta whose resolved size is past the base. git index-pack is the oracle for every pack. A dropped connection
4// has no pack; its oracle is the stated code.
5import { test, beforeAll as before, afterAll as after } from 'vitest';
6import assert from 'node:assert/strict';
7import { spawnSync } from 'node:child_process';
8import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
9import { tmpdir } from 'node:os';
10import { join } from 'node:path';
11import { tmpRoot, makeBareRepo, startGitServer, commitFiles, git, initRepo } from '../server.js';
12import { editFile } from '../../src/edit.js';
13import { readFile, openBlob } from '../../src/read.js';
14import { fetchObject as fetchObjectR } from '../../src/fetch.js';
15import { Tracer } from '../../src/otel.js';
16import { withLimits, DEFAULT_LIMITS } from '../../src/limits.js';
17import { buildPack, encodeObjHeader, spliceDelta, zlibDeflate } from '../../src/pack.js';
18import { hashObject, TYPE } from '../../src/objects.js';
19import { Sha1 } from '../../src/sha1.js';
20import { concat, te } from '../../src/bytes.js';
21import { pkt, FLUSH } from '../../src/pktline.js';
22import { unwrap, unwrapAsync } from '../unwrap.js';
24// Throwing adapters over the Result API (0.2.0): the assertions below are about the throw.
25const fetchObject = (...a) => unwrapAsync(fetchObjectR(...a));
26const FIND = 'M';
27const AUTHOR = { name: 'adv', email: 'adv@x' };
28const FRAME = 8;
29const ENV = { ...Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith('GIT_'))), GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' };
31const contentOf = n => { const b = new Uint8Array(n); if (n) b[0] = 0x4d; for (let i = 1; i < n; i++) b[i] = 0x78; return b; };
33async function liePack(content, declared) {
34 const z = await zlibDeflate(content);
35 const body = concat([Uint8Array.of(0x50, 0x41, 0x43, 0x4b, 0, 0, 0, 2, 0, 0, 0, 1), encodeObjHeader(TYPE.BLOB, declared), z]);
36 return concat([body, new Sha1().update(body).digest()]);
37}
39// index-pack --strict in a fresh bare repo. Returns the status.
40const indexPack = bytes => {
41 const dir = mkdtempSync(join(tmpdir(), 'ply-idx-'));
42 try {
43 spawnSync('git', ['init', '-q', '--bare', dir], { env: ENV });
44 const file = join(dir, 'p.pack');
45 writeFileSync(file, bytes);
46 const r = spawnSync('git', ['index-pack', '--strict', file], { cwd: dir, env: ENV, encoding: 'utf8' });
47 return r.status;
48 } finally { rmSync(dir, { recursive: true, force: true }); }
49};
51const catBlob = (pack, oid) => {
52 const dir = mkdtempSync(join(tmpdir(), 'ply-cat-'));
53 try {
54 spawnSync('git', ['init', '-q', '--bare', dir], { env: ENV });
55 // index-pack of a named file writes only the .idx beside it; --stdin installs the pack where cat-file reads it
56 const r = spawnSync('git', ['index-pack', '--strict', '--stdin'], { cwd: dir, env: ENV, input: pack });
57 assert.equal(r.status, 0, String(r.stderr));
58 const cat = spawnSync('git', ['cat-file', 'blob', oid], { cwd: dir, env: ENV });
59 assert.equal(cat.status, 0, String(cat.stderr));
60 return cat.stdout;
61 } finally { rmSync(dir, { recursive: true, force: true }); }
62};
64// Band-1 frames of `pack`, 8 bytes at a time, after the packfile section. The real server answers everything else.
65function intercept(oid, pack) {
66 return async (url, init) => {
67 const req = init?.body instanceof Uint8Array ? Buffer.from(init.body).toString('latin1') : '';
68 if (!req.includes('command=fetch\n') || req.includes('filter ') || !req.includes(`want ${oid}\n`)) return fetch(url, init);
69 const frames = [unwrap(pkt(te.encode('packfile\n')))];
70 for (let i = 0; i < pack.length; i += FRAME) frames.push(unwrap(pkt(concat([Uint8Array.of(1), pack.subarray(i, i + FRAME)]))));
71 frames.push(FLUSH);
72 let i = 0;
73 const stream = new ReadableStream({
74 pull(c) { if (i >= frames.length) { c.close(); return; } c.enqueue(frames[i++]); },
75 }, { highWaterMark: 0 });
76 return new Response(stream, { status: 200, headers: { 'content-type': 'application/x-git-upload-pack-result' } });
77 };
78}
80let root, cleanup, srv;
81before(async () => { ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root); });
82after(async () => { await srv.close(); cleanup(); });
84test('declared size one under, at, and one over the inflated length, on read and edit', { timeout: 120_000 }, async () => {
85 const cells = [];
86 for (const n of [0, 1, 64]) for (const declared of (n === 0 ? [0, 1] : [n - 1, n, n + 1])) for (const path of ['read', 'edit']) cells.push({ n, declared, path });
87 assert.equal(cells.length, 16);
88 const packs = new Map();
89 for (const { n, declared } of cells) {
90 const key = `${n}:${declared}`;
91 if (!packs.has(key)) packs.set(key, await liePack(contentOf(n), declared));
92 }
93 for (const [key, pack] of packs) {
94 const [n, declared] = key.split(':').map(Number);
95 const status = indexPack(pack);
96 assert.equal(status === 0, n === declared, `index-pack ${key} status ${status}`);
97 }
98 for (const { n, declared, path } of cells) {
99 const data = contentOf(n);
100 const name = `lie${n}d${declared}${path}`;
101 const bare = makeBareRepo(root, name);
102 const work = join(root, `w${name}`);
103 mkdirSync(work);
104 initRepo(work);
105 const head = commitFiles(work, bare, { 'a.bin': Buffer.from(data) }, 'lie');
106 const oid = hashObject('blob', data);
107 const url = `${srv.url}/${name}.git`;
108 const fetchImpl = intercept(oid, packs.get(`${n}:${declared}`));
109 const opts = { tracer: new Tracer(), fetchImpl };
110 const where = `${path} n=${n} declared=${declared}`;
111 let err = null, got = null;
112 if (path === 'read') {
113 const res = await unwrapAsync(readFile(url, { path: 'a.bin' }, opts));
114 const parts = [];
115 try { for await (const c of res.stream) parts.push(c); got = concat(parts); } catch (e) { err = e; }
116 } else {
117 try { got = await unwrapAsync(editFile(url, { path: 'a.bin', find: FIND, replace: 'N', author: AUTHOR, message: 'lie' }, opts)); }
118 catch (e) { err = e; }
119 }
120 // each cell has its own bare repo; read the ref the operation left, then put main back only so a later
121 // assertion cannot observe a push from this cell
122 const ref = git(bare, ['rev-parse', 'main']).trim();
123 if (ref !== head) git(bare, ['update-ref', 'refs/heads/main', head]);
124 if (n === declared && path === 'read') {
125 assert.equal(err, null, where);
126 assert.ok(Buffer.from(got).equals(Buffer.from(data)), where);
127 } else if (n === declared && path === 'edit' && n === 0) {
128 assert.equal(err?.code, 'match_count', `${where}: ${err?.message}`);
129 assert.equal(ref, head, where);
130 } else if (n === declared) {
131 assert.equal(err, null, `${where}: ${err?.message}`);
132 assert.notEqual(ref, head, where);
133 } else if (declared < n) {
134 assert.equal(err?.code, 'pack_corrupt', `${where}: ${err?.message}`);
135 assert.equal(err.message, `server streamed more than the ${declared} bytes it declared`, where);
136 assert.equal(ref, head, where);
137 } else {
138 assert.equal(err?.code, 'pack_corrupt', `${where}: ${err?.message}`);
139 assert.equal(err.message, path === 'read' ? `inflated ${n} != declared ${declared}` : `inflated ${n} != header size ${declared}`, where);
140 assert.equal(ref, head, where);
141 }
142 }
143});
145test('a header that fails before the body: a raw reset, and a bad signature git also refuses', async () => {
146 const tracer = new Tracer();
147 const parent = tracer.start('p');
148 const oid = 'a'.repeat(40);
149 const limits = DEFAULT_LIMITS;
150 const transport = frames => ({
151 kind: 'http', url: 'http://fake/r.git', tracer, parent: null, limits, signal: null, quirks: {}, objects: new Map(), scrub: null,
152 frames, advertise: async () => { throw new Error('unused'); },
153 });
154 // openBlob returns the header failure (#145); it does not reject
155 const reset = await openBlob(transport(async function* () { throw new Error('reset'); }), oid, { follows: parent });
156 assert.equal(reset.ok, false);
157 assert.equal(reset.error.code, 'pack_corrupt');
158 assert.equal(reset.error.message, 'blob stream: reset');
159 const bad = te.encode('NOPE not a pack!!!!');
160 assert.notEqual(indexPack(bad), 0);
161 const frames = async function* () {
162 yield { kind: 'data', payload: te.encode('packfile\n') };
163 yield { kind: 'data', payload: concat([Uint8Array.of(1), bad]) };
164 yield { kind: 'flush' };
165 };
166 const sig = await openBlob(transport(frames), oid, { follows: parent });
167 assert.equal(sig.ok, false);
168 assert.equal(sig.error.code, 'pack_corrupt');
169 assert.equal(sig.error.message, 'pack: bad signature');
170});
172test('blob:none object cap: one under, at, and one past each size the pack meets', { timeout: 60_000 }, async () => {
173 // A ref-delta's header size is the delta's, not the resolved object's. parsePack refuses a header over maxObjectSize
174 // ('pack object size') before fetchWide can see the resolved bytes ('object size', and only while maxDeltaResult
175 // stays at its default, above the result). Caps are one under, at, and one past the base header, the delta header,
176 // and the resolved length. git index-pack accepts every pack; cat-file is the byte oracle.
177 const bases = [8, 32], expands = [2, 16];
178 const built = [];
179 for (const baseN of bases) for (const extraN of expands) {
180 const base = new Uint8Array(baseN).fill(0x41);
181 const extra = new Uint8Array(extraN).fill(0x42);
182 const result = concat([base, extra]);
183 const { delta } = spliceDelta({ baseLen: baseN, off: baseN, oldLen: 0, insert: extra });
184 const { pack, entries } = await buildPack([{ type: TYPE.BLOB, data: base }, { type: TYPE.BLOB, data: delta, base: { tag: 'ref', oid: hashObject('blob', base) } }]);
185 const oid = hashObject('blob', result);
186 const where = `base ${baseN} +${extraN}`;
187 assert.ok(Buffer.from(catBlob(pack, oid)).equals(Buffer.from(result)), where);
188 const deltaN = entries[1].size;
189 assert.equal(deltaN, delta.length, where);
190 built.push({ pack, result, oid, baseN, extraN, deltaN, resultN: result.length });
191 }
192 const cells = [];
193 for (const b of built) {
194 const caps = new Set();
195 for (const n of [b.baseN, b.deltaN, b.resultN]) for (const cap of [n - 1, n, n + 1]) caps.add(cap);
196 for (const cap of [...caps].sort((a, c) => a - c)) cells.push({ ...b, cap });
197 }
198 assert.equal(cells.length, 32);
199 const tracer = new Tracer();
200 let resolved = 0;
201 for (const b of cells) {
202 const limits = withLimits({ maxObjectSize: b.cap });
203 const frames = async function* () {
204 yield { kind: 'data', payload: te.encode('packfile\n') };
205 for (let i = 0; i < b.pack.length; i += FRAME) yield { kind: 'data', payload: concat([Uint8Array.of(1), b.pack.subarray(i, i + FRAME)]) };
206 yield { kind: 'flush' };
207 };
208 const t = {
209 kind: 'http', url: 'http://fake/r.git', tracer, parent: null, limits, signal: null, quirks: { treeFilter: 'blob:none' }, objects: new Map(), scrub: null, frames,
210 advertise: async () => { throw new Error('unused') },
211 };
212 const where = `base ${b.baseN} +${b.extraN} delta ${b.deltaN} result ${b.resultN} cap ${b.cap}`;
213 const res = await fetchObject(t, { oid: b.oid }).then(v => ({ v }), e => ({ e }));
214 // the base is the first object, so a cap under it never reaches the delta header
215 const refused = b.cap < b.baseN ? ['pack object size', b.baseN] : b.cap < b.deltaN ? ['pack object size', b.deltaN] : b.cap < b.resultN ? ['object size', b.resultN] : null;
216 if (refused) {
217 assert.equal(res.e?.code, 'limit_exceeded', `${where}: ${res.e?.message}`);
218 assert.equal(res.e.what, refused[0], where);
219 assert.equal(res.e.got, refused[1], where);
220 assert.equal(res.e.max, b.cap, where);
221 } else {
222 assert.equal(res.e, undefined, `${where}: ${res.e?.message}`);
223 assert.ok(Buffer.from(res.v.data).equals(Buffer.from(b.result)), where);
224 }
225 if (res.e?.what === 'object size') resolved++;
226 }
227 assert.ok(resolved > 0, 'a cap between the headers and the resolved length');
228});
Diagnostics4 git calls · 221 ms
CallStatusSizeTime0221 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2004.2 KiB216 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=216 git.response.bytes=4287 git.response.cache=MISS git.response.duration_ms=216 git.fetch.wants=0f974146bedd6de3eaca9d10b936c6f236c37a83 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4287 git.pkt.payload.max=4266 git.sideband.band1.bytes=4265 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e30d26bc4e8a31
- Request
- 10c362ad…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format