main
1// Bare repos and packs for the cells that are about to run. A repo whose stamp matches is left in place, so a
2// second sweep does not rebuild a 16 MiB blob. The oracle file is git's own measurements: the pack a blob
3// fetches as, the longest delta chain verify-pack reports, and each pack's object count and inflated sum.
4import { spawn } from 'node:child_process';
5import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, rmSync, writeFileSync } from 'node:fs';
6import { join } from 'node:path';
7import { NO_MAINTENANCE } from '../server.js';
9const STAMP = 'sweep-3';
10const WHEN = 1700000000;
12/** @param {string[]} args @param {{ input?: Buffer | string }} [opts] */
13function git(args, opts = {}) {
14 return new Promise((resolve, reject) => {
15 const child = spawn('git', args, { stdio: ['pipe', 'pipe', 'pipe'] });
16 const out = [];
17 const err = [];
18 let settled = false;
19 const fail = (error) => { if (!settled) { settled = true; reject(error); } };
20 child.stdout.on('data', d => out.push(d));
21 child.stderr.on('data', d => err.push(d));
22 child.on('error', fail);
23 // git closes stdin when it rejects the stream; the close handler reports why.
24 child.stdin.on('error', (error) => { if (error.code !== 'EPIPE') fail(error); });
25 child.on('close', code => {
26 if (settled) return;
27 settled = true;
28 if (code !== 0) reject(new Error(`git ${args.slice(0, 8).join(' ')} exited ${code}: ${Buffer.concat(err).toString().slice(0, 800)}`));
29 else resolve(Buffer.concat(out));
30 });
31 child.stdin.end(opts.input ?? '');
32 });
33}
35/** @param {number} n */
36function urandom(n) {
37 const buf = Buffer.alloc(n);
38 if (n === 0) return buf;
39 const fd = openSync('/dev/urandom', 'r');
40 try {
41 let off = 0;
42 while (off < n) {
43 const got = readSync(fd, buf, off, n - off, null);
44 if (got <= 0) throw new Error('urandom returned no bytes');
45 off += got;
46 }
47 } finally {
48 closeSync(fd);
49 }
50 return buf;
51}
53/** @param {string} dir */
54function stamped(dir) {
55 const path = join(dir, 'sweep-stamp');
56 return existsSync(path) && readFileSync(path, 'utf8') === STAMP;
57}
59/** @param {string} dir */
60async function bare(dir) {
61 rmSync(dir, { recursive: true, force: true });
62 await git(['init', '-q', '--bare', '-b', 'main', dir]);
63 const cfg = [
64 ['uploadpack.allowFilter', 'true'],
65 ['uploadpack.allowAnySHA1InWant', 'true'],
66 ['http.receivepack', 'true'],
67 ['protocol.version', '2'],
68 ...NO_MAINTENANCE,
69 ['core.bigFileThreshold', '2g'],
70 ];
71 for (const [key, value] of cfg) await git(['-C', dir, 'config', key, value]);
72 writeFileSync(join(dir, 'sweep-stamp'), STAMP);
73}
75/** @param {string} ref @param {number} when @param {string} message @param {Buffer[]} ops */
76function commit(ref, when, message, ops) {
77 const head = Buffer.from(`commit ${ref}\ncommitter sweep <sweep@example.com> ${when} +0000\ndata ${Buffer.byteLength(message)}\n${message}\n`);
78 return Buffer.concat([head, ...ops]);
79}
81/** @param {string} path @param {Buffer} data */
82function inline(path, data) {
83 return Buffer.concat([Buffer.from(`M 100644 inline ${path}\ndata ${data.length}\n`), data, Buffer.from('\n')]);
84}
86/** @param {string} dir @param {Buffer} stream */
87async function fastImport(dir, stream) {
88 await git(['-C', dir, 'fast-import', '--quiet', '--done'], { input: Buffer.concat([stream, Buffer.from('done\n')]) });
89}
91/** @param {string} repoGit @param {string} path */
92async function blobPackBytes(repoGit, path) {
93 const oid = (await git(['-C', repoGit, 'rev-parse', `main:${path}`])).toString().trim();
94 const packed = await git(['-C', repoGit, 'pack-objects', '--stdout'], { input: `${oid}\n` });
95 return packed.length;
96}
98/** @param {string} text */
99function chainLength(text) {
100 let max = 0;
101 for (const line of text.split('\n')) {
102 const m = /^chain length = (\d+)/.exec(line);
103 if (m) max = Math.max(max, Number(m[1]));
104 }
105 return max;
106}
108/** @param {string} text */
109function inflatedSum(text) {
110 // verify-pack pads the type column, so a single space after the type only matches `commit`. Split instead.
111 let inflated = 0;
112 let objects = 0;
113 for (const line of text.split('\n')) {
114 const parts = line.trim().split(/\s+/);
115 const size = parts[2];
116 if (parts.length >= 3 && /^[0-9a-f]{40}$/.test(parts[0] ?? '') && size !== undefined && /^\d+$/.test(size)) { inflated += Number(size); objects++; }
117 }
118 return { inflated, objects };
119}
121/** The largest object verify-pack lists as a delta (a line with a depth and a base), sized by cat-file. @param {string} repoGit @param {string} text */
122async function deltaResultMax(repoGit, text) {
123 const oids = [];
124 for (const line of text.split('\n')) {
125 const parts = line.trim().split(/\s+/);
126 if (parts.length >= 7 && /^[0-9a-f]{40}$/.test(parts[0] ?? '') && /^[0-9a-f]{40}$/.test(parts[6] ?? '')) oids.push(parts[0]);
127 }
128 if (oids.length === 0) return 0;
129 const sizes = (await git(['-C', repoGit, 'cat-file', '--batch-check=%(objectsize)'], { input: `${oids.join('\n')}\n` })).toString().trim().split('\n');
130 return Math.max(...sizes.map(Number));
131}
133/**
134 * @param {string} repoGit
135 * @param {string} packPath
136 * @param {number} depth
137 * @param {'ofs' | 'ref'} kind
138 * @param {number} window
139 */
140async function writePack(repoGit, packPath, depth, kind, window) {
141 const list = await git(['-C', repoGit, 'rev-list', '--objects', '--all']);
142 const args = ['-C', repoGit, 'pack-objects', '-q', '--stdout', '--no-reuse-delta', `--depth=${depth}`, `--window=${window}`];
143 if (kind === 'ofs') args.push('--delta-base-offset');
144 const packed = await git(args, { input: list });
145 writeFileSync(packPath, packed);
146 const idx = packPath.replace(/\.pack$/, '.idx');
147 await git(['index-pack', '-o', idx, packPath]);
148 const verified = (await git(['verify-pack', '-v', idx])).toString();
149 return { chain: chainLength(verified), sum: inflatedSum(verified), deltaMax: await deltaResultMax(repoGit, verified) };
150}
152/** @param {string} root @param {string} name @param {Record<string, number>} wires */
153async function buildRepo(root, name, wires) {
154 const dir = join(root, `${name}.git`);
155 if (!stamped(dir)) {
156 await bare(dir);
157 const caps = /^cap-([a-z]+)--(.+)$/.exec(name);
158 const body = repoStream(caps ? caps[2] : name);
159 if (body.length > 0) await fastImport(dir, body);
160 // a server without one capability ply can use; the content is the named fixture's
161 if (caps) for (const [key, value] of CAP_MODES[caps[1]] ?? []) await git(['-C', dir, 'config', key, value]);
162 }
163 // Measured even when the repo was reused: the oracle file is the only copy of the pack size.
164 if (name.startsWith('blob-') && wires[name] === undefined) wires[name] = await blobPackBytes(dir, 'big.bin');
165}
167/** @type {Record<string, [string, string][]>} */
168const CAP_MODES = {
169 nofilter: [['uploadpack.allowFilter', 'false']],
170 nosha: [['uploadpack.allowAnySHA1InWant', 'false'], ['uploadpack.allowReachableSHA1InWant', 'false'], ['uploadpack.allowTipSHA1InWant', 'false']],
171 reachable: [['uploadpack.allowAnySHA1InWant', 'false'], ['uploadpack.allowReachableSHA1InWant', 'true']],
172 nopush: [['http.receivepack', 'false']],
173};
175/** @param {number} lines @param {number} version */
176function thinText(lines, version) {
177 const out = [];
178 for (let i = 0; i < lines; i++) out.push(i % 97 === version ? `line ${i} edited in v${version}` : `line ${String(i).padStart(5, '0')} ${'t'.repeat(56)}`);
179 return Buffer.from(`${out.join('\n')}\n`);
180}
182/** @param {string} name */
183function repoStream(name) {
184 if (name === 'empty') return Buffer.alloc(0);
185 if (name === 'thin-remote' || name === 'thin-local') {
186 // the same first commit in both (same bytes, same time: same oid); the local one has a second that edits the file
187 const parts = [commit('refs/heads/main', WHEN, 'v1', [inline('t.txt', thinText(1024, 1))])];
188 if (name === 'thin-local') parts.push(commit('refs/heads/main', WHEN + 1, 'v2', [inline('t.txt', thinText(1024, 2))]));
189 return Buffer.concat(parts);
190 }
191 if (name.startsWith('octo-')) {
192 // one commit with N parents, each its own root commit (a distinct empty-tree commit), reachable only through it
193 const n = Number(name.slice('octo-'.length));
194 const parts = [];
195 for (let i = 0; i < n; i++) parts.push(Buffer.from(`reset refs/heads/tmp\n`), Buffer.concat([Buffer.from(`commit refs/heads/tmp\nmark :${i + 1}\ncommitter sweep <sweep@example.com> ${WHEN} +0000\ndata ${String(i).length + 1}\nr${i}\n`)]));
196 const merges = Array.from({ length: n - 1 }, (_, i) => `merge :${i + 2}\n`).join('');
197 parts.push(Buffer.from(`commit refs/heads/main\ncommitter sweep <sweep@example.com> ${WHEN + 1} +0000\ndata 4\nocto\nfrom :1\n${merges}`), inline('f.txt', Buffer.from('o\n')));
198 parts.push(Buffer.from('reset refs/heads/tmp\nfrom 0000000000000000000000000000000000000000\n\n'));
199 return Buffer.concat(parts);
200 }
201 if (name.startsWith('branches-')) {
202 // main plus N - 1 more branches at the same commit: a push of all of them is N commands and N + 1 report lines
203 const n = Number(name.slice('branches-'.length));
204 const parts = [commit('refs/heads/main', WHEN, 'branches', [inline('f.txt', Buffer.from('b\n'))])];
205 for (let i = 0; i < n - 1; i++) parts.push(Buffer.from(`reset refs/heads/b${String(i).padStart(6, '0')}\nfrom refs/heads/main\n\n`));
206 return Buffer.concat(parts);
207 }
208 if (name.startsWith('blob-')) {
209 const size = Number(name.slice('blob-'.length));
210 return commit('refs/heads/main', WHEN, 'blob', [inline('big.bin', urandom(size)), inline('small.txt', Buffer.from('hello\n'))]);
211 }
212 if (name.startsWith('tree-')) {
213 const entries = Number(name.slice('tree-'.length));
214 if (entries === 0) return commit('refs/heads/main', WHEN, 'empty', []);
215 const blob = Buffer.concat([Buffer.from('blob\nmark :1\ndata 1\nx\n')]);
216 const ops = [];
217 for (let i = 0; i < entries; i++) ops.push(Buffer.from(`M 100644 :1 e${String(i).padStart(7, '0')}\n`));
218 return Buffer.concat([blob, commit('refs/heads/main', WHEN, 'tree', ops)]);
219 }
220 if (name.startsWith('depth-')) {
221 const depth = Number(name.slice('depth-'.length));
222 const path = depth === 1 ? 'f.txt' : `${Array.from({ length: depth - 1 }, () => 'd').join('/')}/f.txt`;
223 return commit('refs/heads/main', WHEN, 'depth', [inline(path, Buffer.from('deep\n'))]);
224 }
225 if (name.startsWith('comp-')) {
226 const length = Number(name.slice('comp-'.length));
227 return commit('refs/heads/main', WHEN, 'comp', [inline('a'.repeat(length), Buffer.from('c'))]);
228 }
229 if (name.startsWith('log-')) {
230 const n = Number(name.slice('log-'.length));
231 const parts = [];
232 for (let i = 0; i < n; i++) parts.push(commit('refs/heads/main', WHEN + i, `c${i}`, [inline('f.txt', Buffer.from(`${i}\n`))]));
233 return Buffer.concat(parts);
234 }
235 if (name.startsWith('refs-')) {
236 const n = Number(name.slice('refs-'.length));
237 const parts = [commit('refs/heads/main', WHEN, 'refs', [inline('f.txt', Buffer.from('r\n'))])];
238 for (let i = 0; i < n - 1; i++) parts.push(Buffer.from(`reset refs/tags/t${String(i).padStart(6, '0')}\nfrom refs/heads/main\n\n`));
239 return Buffer.concat(parts);
240 }
241 if (name.startsWith('mb-')) {
242 const k = Number(name.slice('mb-'.length));
243 const parts = [commit('refs/heads/main', WHEN, 'base', [inline('a.txt', Buffer.from('a\n')), inline('b.txt', Buffer.from('b\n'))])];
244 parts.push(Buffer.from('reset refs/heads/feat\nfrom refs/heads/main\n\n'));
245 for (let i = 0; i < k; i++) parts.push(commit('refs/heads/main', WHEN + 10 + i, `m${i}`, [inline('a.txt', Buffer.from(`a${i}\n`))]));
246 for (let i = 0; i < k; i++) parts.push(commit('refs/heads/feat', WHEN + 10 + i, `f${i}`, [inline('b.txt', Buffer.from(`b${i}\n`))]));
247 return Buffer.concat(parts);
248 }
249 if (name.startsWith('mblob-')) {
250 const size = Number(name.slice('mblob-'.length));
251 const trio = mergeTrio(size);
252 const parts = [commit('refs/heads/main', WHEN, 'base', [inline('big.txt', trio.base)])];
253 parts.push(Buffer.from('reset refs/heads/feat\nfrom refs/heads/main\n\n'));
254 parts.push(commit('refs/heads/main', WHEN + 1, 'ours', [inline('big.txt', trio.ours)]));
255 parts.push(commit('refs/heads/feat', WHEN + 2, 'theirs', [inline('big.txt', trio.theirs)]));
256 return Buffer.concat(parts);
257 }
258 if (name.startsWith('edit-')) {
259 const size = Number(name.slice('edit-'.length));
260 return commit('refs/heads/main', WHEN, 'edit', [inline('e.txt', editBody(size))]);
261 }
262 if (name === 'push-empty') return commit('refs/heads/main', WHEN, 'empty', []);
263 if (name === 'inflate') {
264 const ops = [];
265 for (let i = 0; i < 6; i++) ops.push(inline(`b${i}.bin`, Buffer.alloc(1000, 0x30 + i)));
266 return commit('refs/heads/main', WHEN, 'inflate', ops);
267 }
268 if (name.startsWith('tiny-')) {
269 // n distinct 7-byte blobs, 1000 to a directory
270 const n = Number(name.slice('tiny-'.length));
271 const parts = [];
272 for (let i = 0; i < n; i++) parts.push(Buffer.from(`blob\nmark :${i + 1}\ndata 7\n${String(i).padStart(7, '0')}\n`));
273 const ops = [];
274 for (let i = 0; i < n; i++) ops.push(Buffer.from(`M 100644 :${i + 1} d${Math.floor(i / 1000)}/f${i}\n`));
275 parts.push(commit('refs/heads/main', WHEN, 'tiny', ops));
276 return Buffer.concat(parts);
277 }
278 if (name === 'chain-src') return chainStream(80, 80);
279 if (name === 'chain-long') return chainStream(300, 1000);
280 throw new Error(`no fixture builder for ${name}`);
281}
283/** @param {number} versions @param {number} lines */
284function chainStream(versions, lines) {
285 const body = Array.from({ length: lines }, (_, i) => `line ${String(i).padStart(4, '0')} ${'x'.repeat(48)}`);
286 const parts = [];
287 for (let v = 0; v < versions; v++) {
288 body[v % lines] = `line ${String(v % lines).padStart(4, '0')} changed ${String(v).padStart(4, '0')} ${'y'.repeat(40)}`;
289 parts.push(commit('refs/heads/main', WHEN + v, `v${v}`, [inline('f.txt', Buffer.from(`${body.join('\n')}\n`))]));
290 }
291 return Buffer.concat(parts);
292}
294/** @param {number} size */
295function editBody(size) {
296 const needle = Buffer.from('NEEDLE\n');
297 if (size < needle.length) throw new Error(`edit fixture ${size} is shorter than the needle`);
298 const left = Math.floor((size - needle.length) / 2);
299 const right = size - needle.length - left;
300 return Buffer.concat([Buffer.alloc(left, 0x71), needle, Buffer.alloc(right, 0x72)]);
301}
303/** @param {number} size */
304function mergeTrio(size) {
305 // Three lines: a changed first line, one unchanged line, a changed last line. Adjacent edits conflict
306 // ("both sides changed the same lines"), and a middle of millions of short lines makes the diff itself
307 // the cost. One unchanged line keeps the merge to three records whatever the file size is.
308 const head = 5;
309 const tail = 5;
310 if (size < head + tail + 2) throw new Error(`merge fixture ${size} needs two lines and a separator`);
311 const middleLen = size - head - tail;
312 const mid = Buffer.alloc(middleLen, 0x7a);
313 mid[middleLen - 1] = 0x0a;
314 const file = (h, t) => Buffer.concat([Buffer.from(h), mid, Buffer.from(t)]);
315 return { base: file('AAAA\n', 'AAAA\n'), ours: file('OOOO\n', 'AAAA\n'), theirs: file('AAAA\n', 'TTTT\n') };
316}
318/**
319 * @param {string} root
320 * @param {{ repo?: string, pack?: string, local?: string, store?: string }[]} cells
321 */
322export async function ensureFixtures(root, cells) {
323 mkdirSync(root, { recursive: true });
324 const oraclePath = join(root, 'oracle.json');
325 /** @type {{ wires: Record<string, number>, chains: Record<string, number>, sums: Record<string, { inflated: number, objects: number }>, deltaMax: Record<string, number> }} */
326 const oracle = { wires: {}, chains: {}, sums: {}, deltaMax: {}, ...(existsSync(oraclePath) ? JSON.parse(readFileSync(oraclePath, 'utf8')) : {}) };
327 oracle.deltaMax ??= {};
328 const repos = [...new Set(cells.flatMap(c => [c.repo, c.local, c.store]).filter(Boolean))];
329 for (const name of repos) await buildRepo(root, name, oracle.wires);
330 const packs = [...new Set(cells.map(c => c.pack).filter(Boolean))];
331 if (packs.length > 0) {
332 mkdirSync(join(root, 'packs'), { recursive: true });
333 if (packs.some(id => !id.startsWith('long-') && !id.startsWith('tiny-') && id !== 'inflate')) await buildRepo(root, 'chain-src', oracle.wires);
334 for (const id of packs.filter(p => p.startsWith('tiny-'))) await buildRepo(root, id, oracle.wires);
335 if (packs.some(id => id.startsWith('long-'))) await buildRepo(root, 'chain-long', oracle.wires);
336 if (packs.includes('inflate')) await buildRepo(root, 'inflate', oracle.wires);
337 for (const id of packs) {
338 const packPath = join(root, 'packs', `${id}.pack`);
339 if (existsSync(packPath) && oracle.chains[id] !== undefined && oracle.sums[id] !== undefined && oracle.deltaMax[id] !== undefined) continue;
340 // writePack runs git index-pack on the pack: git accepts every pack a cell parses
341 const built = await buildOnePack(root, id, packPath);
342 oracle.chains[id] = built.chain;
343 oracle.sums[id] = built.sum;
344 oracle.deltaMax[id] = built.deltaMax;
345 }
346 }
347 writeFileSync(oraclePath, `${JSON.stringify(oracle)}\n`);
348 return oracle;
349}
351/** @param {string} root @param {string} id @param {string} packPath */
352async function buildOnePack(root, id, packPath) {
353 if (id === 'inflate' || id.startsWith('tiny-')) return writePack(join(root, `${id}.git`), packPath, 0, 'ofs', 0);
354 const long = id.startsWith('long-');
355 const rest = long ? id.slice('long-'.length) : id;
356 const m = /^d(\d+)-(ofs|ref)$/.exec(rest);
357 if (!m) throw new Error(`bad pack id ${id}`);
358 const repo = join(root, long ? 'chain-long.git' : 'chain-src.git');
359 const window = long ? 300 : 80;
360 return writePack(repo, packPath, Number(m[1]), m[2] === 'ofs' ? 'ofs' : 'ref', window);
361}
Diagnostics4 git calls · 330 ms
CallStatusSizeTime0330 ms
discover200152 B7 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit7 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=316 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=7 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2005.3 KiB316 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=5383 git.response.cache=MISS git.response.duration_ms=316 git.fetch.wants=82d9ab14ee1a86deb1ee2d2c03b5681e941b6172 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5383 git.pkt.payload.max=5362 git.sideband.band1.bytes=5361 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e33db19cdedafe
- Request
- e95a365c…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format