main
1// Sweep driver. `npm run sweep` (and `--ci`) runs the cells at or under 16 MiB and 65_536 entries.
2// `npm run sweep -- --full` runs the rest as well: the nightly workflow does that.
3// Each cell is a fresh node process, once on the default heap and once with --max-old-space-size=128; a workerd cell
4// is one fresh workerd process instead (heap 'workerd'). Wire cells reach git through shim.mjs (slow, v0/v1, re-framed).
5// Cells that push, edit, or merge copy the bare repo first and the parent deletes the copy.
6// A result that is not the desired one fails the run unless test/sweep/ratchet.json records it. A recorded
7// break that now matches the desired result fails too: the ratchet only goes down, by deleting the line.
8import { execFileSync, spawn, spawnSync } from 'node:child_process';
9import { createHash } from 'node:crypto';
10import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
11import { dirname, join } from 'node:path';
12import { fileURLToPath } from 'node:url';
13import { DEFAULT_LIMITS, LIMIT_OBJECT_OVERHEAD } from '../../src/limits.js';
14import { startGitServer } from '../server.js';
15import { ensureFixtures } from './fixtures.mjs';
16import { allCells, selectCells } from './matrix.mjs';
17import { startShim } from './shim.mjs';
18import { build as buildWorker } from '../workerd/build.mjs';
20const HERE = dirname(fileURLToPath(import.meta.url));
21const ROOT = join(HERE, '..', '..');
22const HOOK = join(ROOT, 'test', 'ts-resolve.mjs');
23const CELL = join(HERE, 'cell.mjs');
24const RATCHET_PATH = join(HERE, 'ratchet.json');
25const HEAP_FLAG = 128;
26const TIMEOUT_MS = Number(process.env.SWEEP_TIMEOUT_MS ?? 240_000);
27const PAR = Math.max(1, Number(process.env.SWEEP_PAR ?? 2));
28const WORKERD = [process.env.WORKERD, join(ROOT, 'node_modules', '.bin', 'workerd')].filter(Boolean).find(p => existsSync(p)) ?? null;
30const argv = process.argv.slice(2);
31const flag = (name) => argv.includes(name);
32const option = (name) => {
33 const i = argv.indexOf(name);
34 return i >= 0 ? argv[i + 1] : null;
35};
37if (flag('--ci') && flag('--full')) {
38 process.stderr.write('pass one of --ci or --full\n');
39 process.exit(2);
40}
41const full = flag('--full');
42const filter = option('--filter');
43const outDir = option('--out') ?? join(ROOT, 'reports', 'sweep');
44const listOnly = flag('--list');
46const selected = selectCells({ ci: full ? false : true, full, filter });
47if (listOnly) {
48 process.stdout.write(`${selected.length} cells (${selected.reduce((n, c) => n + (c.runtime === 'workerd' ? 1 : 2), 0)} runs)\n`);
49 for (const cell of selected) process.stdout.write(`${cell.id}\n`);
50 process.exit(0);
51}
53/** @param {Record<string, unknown>} view @param {Record<string, unknown>} spec */
54function covers(view, spec) {
55 return Object.entries(spec).every(([key, value]) => view[key] === value);
56}
58/** @param {Record<string, unknown>} result */
59function viewOf(result) {
60 /** @type {Record<string, unknown>} */
61 const view = { status: result.status };
62 for (const key of ['code', 'what', 'got', 'max']) if (result[key] != null) view[key] = result[key];
63 return view;
64}
66/** @param {Uint8Array | string} data */
67const pkt = data => {
68 const b = Buffer.from(data);
69 return Buffer.concat([Buffer.from((b.length + 4).toString(16).padStart(4, '0')), b]);
70};
72// git's upload-pack, asked the cell's fetch directly: the lines of each section it answers before the packfile.
73/** @param {string} url @param {{ wants: string[], haves?: string[], deepen?: number, done?: boolean, filter?: string }} req */
74async function gitSections(url, req) {
75 const lines = [pkt('command=fetch\n'), Buffer.from('0001'), ...req.wants.map(w => pkt(`want ${w}\n`)), ...(req.haves ?? []).map(h => pkt(`have ${h}\n`))];
76 if (req.filter) lines.push(pkt(`filter ${req.filter}\n`));
77 if (req.deepen) lines.push(pkt(`deepen ${req.deepen}\n`));
78 if (req.done !== false) lines.push(pkt('done\n'));
79 lines.push(Buffer.from('0000'));
80 const res = await fetch(`${url}/git-upload-pack`, { method: 'POST', body: Buffer.concat(lines), headers: { 'Git-Protocol': 'version=2', 'Content-Type': 'application/x-git-upload-pack-request' } });
81 const body = Buffer.from(await res.arrayBuffer());
82 /** @type {Record<string, number>} */
83 const counts = {};
84 let section = null;
85 for (let p = 0; p + 4 <= body.length;) {
86 const len = parseInt(body.subarray(p, p + 4).toString(), 16);
87 if (len < 4) { p += 4; if (len === 1) section = null; if (len === 0) break; continue; }
88 const line = body.subarray(p + 4, p + len).toString().replace(/\n$/, '');
89 p += len;
90 if (section === null) { section = line; if (line === 'packfile') break; counts[line] = 0; continue; }
91 counts[section] += 1;
92 }
93 return counts;
94}
96/** @param {string} repo @param {string} rev */
97const revParse = (repo, rev) => gitText(repo, ['rev-parse', rev]);
99/**
100 * @param {import('./matrix.mjs').Cell} cell
101 * @param {{ wires: Record<string, number>, chains: Record<string, number>, sums: Record<string, { inflated: number, objects: number }>, deltaMax: Record<string, number> }} oracle
102 * @param {{ root: string, server: string, shim: string }} ctx
103 */
104async function decorate(cell, oracle, ctx) {
105 const next = { ...cell, limits: cell.limits ? { ...cell.limits } : undefined, expect: cell.expect ? { ...cell.expect } : undefined };
106 const repoDir = cell.repo ? join(ctx.root, `${cell.repo}.git`) : null;
107 const refusedAs = (/** @type {Record<string, unknown>} */ spec) => ({ status: 'refused', code: 'limit_exceeded', ...spec });
108 if (cell.expectFrom === 'wire') {
109 const wire = oracle.wires[cell.repo];
110 if (typeof wire !== 'number') throw new Error(`${cell.id}: no wire size for ${cell.repo}`);
111 const cap = cell.limits?.maxPackBytes ?? DEFAULT_LIMITS.maxPackBytes;
112 next.wire = wire;
113 next.expect = wire > cap
114 ? { status: 'refused', code: 'limit_exceeded', what: 'pack bytes' }
115 : { status: 'ok' };
116 } else if (cell.expectFrom === 'chain' || cell.expectFrom === 'chain-limit') {
117 const chain = oracle.chains[cell.pack];
118 if (typeof chain !== 'number') throw new Error(`${cell.id}: no chain length for ${cell.pack}`);
119 const cap = cell.limits?.maxDeltaChain ?? DEFAULT_LIMITS.maxDeltaChain;
120 next.chain = chain;
121 next.expect = chain > cap
122 ? { status: 'refused', code: 'limit_exceeded', what: 'delta chain depth' }
123 : { status: 'ok' };
124 } else if (cell.expectFrom === 'charged') {
125 // what parsePack charges: verify-pack's inflated sum plus LIMIT_OBJECT_OVERHEAD per object it lists
126 const m = oracle.sums?.[cell.pack];
127 if (!m) throw new Error(`${cell.id}: no verify-pack sum for ${cell.pack}`);
128 const charged = m.inflated + m.objects * LIMIT_OBJECT_OVERHEAD;
129 const cap = cell.limits?.maxInflatedTotal ?? DEFAULT_LIMITS.maxInflatedTotal;
130 next.inflated = charged;
131 next.expect = charged > cap ? { status: 'refused', code: 'limit_exceeded', what: 'pack inflated bytes' } : { status: 'ok' };
132 } else if (cell.expectFrom === 'objects') {
133 // verify-pack's object count is the boundary; capDelta sets the cap beside it
134 const count = oracle.sums?.[cell.pack]?.objects;
135 if (typeof count !== 'number') throw new Error(`${cell.id}: no verify-pack count for ${cell.pack}`);
136 if (typeof cell.capDelta === 'number') next.limits = { ...cell.limits, maxPackObjects: count + cell.capDelta };
137 const cap = next.limits?.maxPackObjects ?? DEFAULT_LIMITS.maxPackObjects;
138 next.objects = count;
139 next.expect = count > cap ? refusedAs({ what: 'pack objects', got: count, max: cap }) : { status: 'ok' };
140 } else if (cell.expectFrom === 'delta-result') {
141 const R = oracle.deltaMax?.[cell.pack];
142 if (typeof R !== 'number' || R <= 0) throw new Error(`${cell.id}: no delta in ${cell.pack}`);
143 next.limits = { ...cell.limits, maxDeltaResult: R + (cell.capDelta ?? 0) };
144 next.deltaMax = R;
145 next.expect = R > next.limits.maxDeltaResult ? refusedAs({ what: 'delta result size', got: R }) : { status: 'ok' };
146 } else if (cell.expectFrom === 'harvest') {
147 // what a full window of this repository holds in commits and trees, by git's sizes
148 let sum = 0;
149 for (const line of gitText(repoDir, ['cat-file', '--batch-all-objects', '--batch-check=%(objecttype) %(objectsize)']).split('\n')) {
150 const [type, size] = line.split(' ');
151 if (type === 'commit' || type === 'tree') sum += Number(size);
152 }
153 next.limits = { ...cell.limits, maxHarvestBytes: sum + (cell.capDelta ?? 0) };
154 next.harvest = sum;
155 next.expect = { status: 'ok' };
156 } else if (cell.expectFrom === 'sections') {
157 const revs = gitText(repoDir, ['rev-list', 'main']).split('\n');
158 next.wants = cell.wantCount ? revs.slice(0, cell.wantCount) : [revs[0]];
159 next.haves = cell.haveCount ? revs.slice(1, 1 + cell.haveCount) : [];
160 const counts = await gitSections(`${ctx.server}/${cell.repo}.git`, { wants: next.wants, haves: next.haves, deepen: cell.deepen, done: cell.done, filter: cell.filter });
161 const cap = cell.limits?.maxSectionLines ?? DEFAULT_LIMITS.maxSectionLines;
162 next.gitSections = counts;
163 const acks = counts.acknowledgments ?? 0;
164 const shallow = counts['shallow-info'] ?? 0;
165 if (acks > cap) next.expect = refusedAs({ what: 'acknowledgment lines', got: cap + 1, max: cap });
166 else if (shallow > cap) next.expect = refusedAs({ what: 'shallow-info lines', got: cap + 1, max: cap });
167 else next.expect = { status: 'ok' };
168 } else if (cell.expectFrom === 'git-clone') {
169 // git's client through the same shim URL: what it accepts, ply accepts
170 const dest = join(ctx.root, `clone-${cell.id.replace(/[^a-zA-Z0-9]+/g, '_')}`);
171 rmSync(dest, { recursive: true, force: true });
172 // async: the shim and the server answering this clone run in this process
173 const r = await new Promise(resolve => {
174 const child = spawn('git', ['-c', 'protocol.version=2', 'clone', '-q', '--bare', `${ctx.shim}/~${cell.shim}/${cell.repo}.git`, dest], { stdio: ['ignore', 'ignore', 'pipe'], env: { ...process.env, GIT_TERMINAL_PROMPT: '0' } });
175 let stderr = '';
176 child.stderr.on('data', d => { stderr += d; });
177 child.on('close', status => resolve({ status, stderr }));
178 });
179 rmSync(dest, { recursive: true, force: true });
180 next.gitClone = r.status === 0 ? 'ok' : r.stderr.trim().split('\n').at(-1)?.slice(0, 200) ?? `exit ${r.status}`;
181 next.expect = r.status === 0 ? { status: 'ok' } : { status: 'refused', code: 'protocol' };
182 } else if (cell.expectFrom === 'thin' || cell.expectFrom === 'thin-missing') {
183 const remote = join(ctx.root, 'thin-remote.git');
184 next.wants = [revParse(repoDir, 'main')];
185 next.haves = [revParse(remote, 'main')];
186 const list = gitText(repoDir, ['rev-list', '--objects', 'main', `^${next.haves[0]}`]).split('\n').map(l => l.slice(0, 40)).sort();
187 const hash = createHash('sha1');
188 for (const oid of list) hash.update(oid);
189 next.thinOids = hash.digest('hex');
190 if (cell.expectFrom === 'thin') next.expect = { status: 'ok' };
191 } else if (cell.expectFrom === 'inflated-at' || cell.expectFrom === 'inflated-under') {
192 // the aggregate is the inflated sum plus LIMIT_OBJECT_OVERHEAD per object (#142)
193 const m = oracle.sums?.[cell.pack];
194 if (!m) throw new Error(`${cell.id}: inflated sum was not measured`);
195 const sum = m.inflated + m.objects * LIMIT_OBJECT_OVERHEAD;
196 const max = cell.expectFrom === 'inflated-at' ? sum : sum - 1;
197 next.limits = { ...cell.limits, maxInflatedTotal: max };
198 next.expect = cell.expectFrom === 'inflated-under'
199 ? { status: 'refused', code: 'limit_exceeded', what: 'pack inflated bytes' }
200 : { status: 'ok' };
201 next.inflated = sum;
202 }
203 if (cell.local) next.localDir = join(ctx.root, `${cell.local}.git`);
204 if (cell.store) next.storeDir = join(ctx.root, `${cell.store}.git`);
205 if (!next.expect) throw new Error(`${cell.id}: no expect`);
206 return next;
207}
209/** @param {string} repo @param {string[]} args */
210function gitText(repo, args) {
211 return execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8', maxBuffer: 64 << 20 }).trimEnd();
212}
214/** @param {ReturnType<typeof decorate>} cell @param {string} repoDir @param {Record<string, unknown>} result */
215function checkOracle(cell, repoDir, result) {
216 if (result.status !== 'ok') return null;
217 if (cell.op === 'readFile') {
218 const rev = `${cell.ref ?? 'main'}:${cell.path}`;
219 const oid = gitText(repoDir, ['rev-parse', rev]);
220 const size = Number(gitText(repoDir, ['cat-file', '-s', oid]));
221 const match = result.blob === oid && result.size === size && result.len === size;
222 return match ? null : `readFile ${rev}: ply ${result.blob} size ${result.size} len ${result.len}, git ${oid} size ${size}`;
223 }
224 if (cell.op === 'listTree') {
225 const spec = cell.path ? `main:${cell.path}` : 'main';
226 const text = gitText(repoDir, ['ls-tree', '--name-only', spec]);
227 const n = text === '' ? 0 : text.split('\n').length;
228 return result.n === n ? null : `listTree ${spec}: ply ${result.n}, git ${n}`;
229 }
230 if (cell.op === 'readLog') {
231 const text = gitText(repoDir, ['rev-list', '--first-parent', 'main']);
232 const count = text === '' ? 0 : text.split('\n').length;
233 const n = Math.min(count, cell.limit);
234 const truncated = count > cell.limit;
235 return result.n === n && result.truncated === truncated ? null : `readLog n${cell.history} l${cell.limit}: ply ${result.n} truncated ${result.truncated}, git ${n} truncated ${truncated}`;
236 }
237 if (cell.op === 'lsRefs') {
238 const text = gitText(repoDir, ['ls-remote', '.']);
239 const n = text === '' ? 0 : text.split('\n').length;
240 return result.n === n ? null : `lsRefs: ply ${result.n}, git ls-remote ${n}`;
241 }
242 if (cell.op === 'merge') {
243 const base = gitText(repoDir, ['merge-base', 'main', 'feat']);
244 return result.base === base ? null : `merge-base: ply ${result.base}, git ${base}`;
245 }
246 if (cell.op === 'pack') {
247 if (typeof cell.objects === 'number' && result.n !== cell.objects) return `pack objects: ply ${result.n}, verify-pack ${cell.objects}`;
248 if (typeof cell.chain !== 'number') return null;
249 return result.maxChain === cell.chain ? null : `pack chain: ply ${result.maxChain}, verify-pack ${cell.chain}`;
250 }
251 if (cell.op === 'fetchPack') {
252 const g = cell.gitSections ?? {};
253 // git's acknowledgments section is the ACKs then `ready`; ply reports the ACKs and the ready flag
254 const acks = Math.max(0, (g.acknowledgments ?? 0) - (result.ready ? 1 : 0));
255 const shallow = g['shallow-info'] ?? 0;
256 return result.acks === acks && result.shallow === shallow ? null : `sections: ply acks ${result.acks} shallow ${result.shallow}, git ${JSON.stringify(g)}`;
257 }
258 if (cell.op === 'fetchThin') {
259 if (result.refDelta < 1) return `thin fetch: no REF_DELTA in the pack, the external path was not exercised`;
260 return result.oidsHash === cell.thinOids ? null : `thin fetch: ply objects ${result.n} differ from git rev-list --objects`;
261 }
262 if (cell.op === 'pushRepo') {
263 const heads = (/** @type {string} */ dir) => gitText(dir, ['for-each-ref', '--format=%(refname) %(objectname)', ...(cell.spec?.all ? ['refs/heads/'] : ['refs/heads/main'])]);
264 const local = heads(cell.localDir);
265 const remote = heads(repoDir);
266 if (local !== remote) return `pushRepo: remote refs differ from local (${remote.split('\n').length} vs ${local.split('\n').length})`;
267 if (cell.thin === true && !(result.thinDeltas > 0)) return 'pushRepo thin: no delta against the remote was sent';
268 const fsck = execFileSync('git', ['-C', repoDir, 'fsck', '--strict', '--no-dangling'], { encoding: 'utf8' });
269 return fsck.trim() === '' ? null : `fsck: ${fsck.trim().slice(0, 200)}`;
270 }
271 if (cell.op === 'rebase') {
272 const count = Number(gitText(repoDir, ['rev-list', '--count', 'main..feat']));
273 const onMain = spawnSync('git', ['-C', repoDir, 'merge-base', '--is-ancestor', 'main', 'feat']).status === 0;
274 if (count !== cell.rebased || !onMain) return `rebase: git sees ${count} commits main..feat (want ${cell.rebased}), main ${onMain ? 'is' : 'is not'} an ancestor of feat`;
275 const fsck = execFileSync('git', ['-C', repoDir, 'fsck', '--strict', '--no-dangling'], { encoding: 'utf8' });
276 return fsck.trim() === '' ? null : `fsck: ${fsck.trim().slice(0, 200)}`;
277 }
278 if (cell.op === 'editFile' || cell.op === 'pushTree') {
279 const fsck = execFileSync('git', ['-C', repoDir, 'fsck', '--strict', '--no-dangling'], { encoding: 'utf8' });
280 return fsck.trim() === '' ? null : `fsck: ${fsck.trim().slice(0, 200)}`;
281 }
282 return null;
283}
285/** @param {number} pid */
286function procCost(pid) {
287 try {
288 const status = readFileSync(`/proc/${pid}/status`, 'utf8');
289 const hwm = Number(/VmHWM:\s+(\d+)/.exec(status)?.[1] ?? 0);
290 const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
291 const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
292 // utime and stime are fields 14 and 15 of /proc/pid/stat, in clock ticks (100 per second on Linux)
293 const ticks = Number(fields[11]) + Number(fields[12]);
294 return { rssPeakMB: Math.round(hwm / 1024), cpuMs: ticks * 10 };
295 } catch {
296 return { rssPeakMB: null, cpuMs: null };
297 }
298}
300let wdPort = 19100 + (process.pid % 200) * 4;
301/** @type {{ dir: string, config: string } | null} */
302let wdBuild = null;
304// A workerd cell: a fresh workerd process serving test/workerd/worker.mjs, one POST /sweep, then its peak RSS (VmHWM)
305// and CPU from /proc before it is killed. Those are the isolate's cost: the process holds nothing else.
306/** @param {Record<string, unknown>} payload */
307async function runWorkerd(payload) {
308 if (!WORKERD) return { status: 'crash', code: null, msg: 'workerd not found (node_modules/.bin/workerd or $WORKERD)' };
309 if (!wdBuild) throw new Error('workerd bundle was not built');
310 const port = wdPort++;
311 const config = join(wdBuild.dir, `config-${port}.capnp`);
312 writeFileSync(config, wdBuild.config.replace(/127\.0\.0\.1:\d+/, `127.0.0.1:${port}`));
313 const child = spawn(WORKERD, ['serve', config], { stdio: ['ignore', 'ignore', 'pipe'] });
314 let err = '';
315 child.stderr.on('data', d => { err = (err + d).slice(-2000); });
316 const base = `http://127.0.0.1:${port}`;
317 try {
318 let up = false;
319 for (let i = 0; i < 200 && !up; i++) {
320 try { up = (await fetch(`${base}/ping`)).ok; } catch { await new Promise(r => setTimeout(r, 50)); }
321 }
322 if (!up) return { status: 'crash', code: null, msg: `workerd did not answer: ${err.slice(-300)}` };
323 const t0 = performance.now();
324 const res = await fetch(`${base}/sweep`, { method: 'POST', body: JSON.stringify(payload), signal: AbortSignal.timeout(TIMEOUT_MS) });
325 const out = /** @type {Record<string, unknown>} */ (await res.json());
326 return { ...out, ms: Math.round(performance.now() - t0), ...procCost(child.pid ?? 0) };
327 } catch (e) {
328 return { status: 'crash', code: null, msg: `workerd: ${e instanceof Error ? e.message : String(e)} ${err.slice(-300)}` };
329 } finally {
330 child.kill('SIGKILL');
331 rmSync(config, { force: true });
332 }
333}
335/** @param {any} cell @param {string} origin @param {string} shimOrigin @param {string} root */
336function urlOf(cell, repoName, origin, shimOrigin) {
337 if (!repoName) return null;
338 return cell.shim ? `${shimOrigin}/~${cell.shim}/${repoName}.git` : `${origin}/${repoName}.git`;
339}
341/**
342 * @param {Awaited<ReturnType<typeof decorate>>} cell
343 * @param {'default' | '128' | 'workerd'} heap
344 * @param {{ server: string, shim: string }} origins
345 * @param {string} root
346 */
347function runOne(cell, heap, origins, root) {
348 return new Promise(resolve => {
349 let repoName = cell.repo ?? null;
350 let copyDir = null;
351 if (cell.writes && cell.repo) {
352 repoName = `w-${cell.id.replace(/[^a-zA-Z0-9]+/g, '_')}-${heap}`;
353 copyDir = join(root, `${repoName}.git`);
354 rmSync(copyDir, { recursive: true, force: true });
355 cpSync(join(root, `${cell.repo}.git`), copyDir, { recursive: true });
356 }
357 const payload = {
358 ...cell,
359 url: urlOf(cell, repoName, origins.server, origins.shim),
360 pack: cell.pack ? join(root, 'packs', `${cell.pack}.pack`) : null,
361 local: cell.localDir ?? null,
362 store: cell.storeDir ?? null,
363 };
364 if (heap === 'workerd') {
365 runWorkerd(payload).then(result => finish(result));
366 return;
367 }
368 const args = ['--import', HOOK];
369 if (heap === '128') args.push(`--max-old-space-size=${HEAP_FLAG}`);
370 args.push(CELL, JSON.stringify(payload));
371 const child = spawn(process.execPath, args, { stdio: ['ignore', 'pipe', 'pipe'] });
372 let out = '';
373 let err = '';
374 let timedOut = false;
375 const limit = cell.timeoutMs ?? TIMEOUT_MS;
376 const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, limit);
377 child.stdout.on('data', d => { out += d; });
378 child.stderr.on('data', d => { err = (err + d).slice(-4000); });
379 child.on('close', (code, signal) => {
380 clearTimeout(timer);
381 /** @type {Record<string, any>} */
382 let result;
383 const line = out.trim().split('\n').filter(Boolean).at(-1);
384 let parsed = null;
385 if (line) {
386 try { parsed = JSON.parse(line); } catch { parsed = null; }
387 }
388 const oom = /JavaScript heap out of memory|Allocation failed|heap limit/i.test(err);
389 if (timedOut) result = { status: 'hang', code: null, msg: `killed after ${limit}ms` };
390 else if (parsed && !oom) result = parsed;
391 else if (oom) result = { status: 'oom', code: null, msg: err.trim().split('\n').at(-1) ?? 'oom' };
392 else result = { status: 'crash', code: null, msg: `exit ${code} signal ${signal}: ${(line ?? err).slice(0, 300)}` };
393 finish(result);
394 });
395 /** @param {Record<string, any>} result */
396 function finish(result) {
397 // Merge-base is read on the untouched fixture. A merge writes a merge commit into the copy, and
398 // merge-base on that copy is a different question. edit and push are judged by fsck of the copy.
399 const oracleDir = ['editFile', 'pushTree', 'pushRepo', 'rebase'].includes(cell.op) ? copyDir : (cell.repo ? join(root, `${cell.repo}.git`) : null);
400 if (result.status === 'ok' && (cell.op === 'pack' || oracleDir)) {
401 try {
402 const problem = checkOracle(cell, cell.op === 'pack' ? root : oracleDir, result);
403 if (problem) result.oracle = problem;
404 } catch (e) {
405 result.oracle = `oracle: ${e instanceof Error ? e.message : String(e)}`;
406 }
407 }
408 if (copyDir) rmSync(copyDir, { recursive: true, force: true });
409 result.id = cell.id;
410 result.heap = heap;
411 result.param = cell.param;
412 result.op = cell.op;
413 result.expect = cell.expect;
414 resolve(result);
415 }
416 });
417}
419/** @param {unknown[]} items @param {number} n @param {(item: unknown) => Promise<void>} fn */
420async function pool(items, n, fn) {
421 let next = 0;
422 const workers = Array.from({ length: Math.min(n, items.length) }, async () => {
423 while (next < items.length) {
424 const item = items[next];
425 next += 1;
426 await fn(item);
427 }
428 });
429 await Promise.all(workers);
430}
432function markdown(results, failures, stale) {
433 const by = new Map();
434 for (const r of results) by.set(r.status, (by.get(r.status) ?? 0) + 1);
435 const lines = [
436 '# limit sweep',
437 '',
438 `${results.length} runs. ${[...by.entries()].map(([k, v]) => `${k} ${v}`).join(', ')}.`,
439 '',
440 ];
441 if (failures.length === 0 && stale.length === 0) lines.push('No divergence outside the ratchet.', '');
442 if (failures.length) {
443 lines.push('## Divergences', '', '| cell | heap | expect | actual |', '|---|---|---|---|');
444 for (const f of failures) lines.push(`| \`${f.id}\` | ${f.heap} | ${JSON.stringify(f.expect)} | ${JSON.stringify(viewOf(f))} ${f.oracle ?? ''} |`);
445 lines.push('');
446 }
447 if (stale.length) {
448 lines.push('## Ratchet lines the cell no longer hits', '');
449 for (const s of stale) lines.push(`- \`${s.id}\` heap ${s.heap}: delete it from test/sweep/ratchet.json`);
450 lines.push('');
451 }
452 const ranked = [...results].sort((a, b) => (b.rssPeakMB ?? 0) - (a.rssPeakMB ?? 0)).slice(0, 20);
453 lines.push('## Peak RSS (top 20)', '', '| cell | heap | status | RSS MiB | ArrayBuffers MiB | heap MiB | ms |', '|---|---|---|---:|---:|---:|---:|');
454 for (const r of ranked) lines.push(`| \`${r.id}\` | ${r.heap} | ${r.status} | ${r.rssPeakMB ?? ''} | ${r.abPeakMB ?? ''} | ${r.heapPeakMB ?? ''} | ${r.ms ?? ''} |`);
455 lines.push('');
456 return lines.join('\n');
457}
459const ratchet = JSON.parse(readFileSync(RATCHET_PATH, 'utf8'));
460const known = new Map(ratchet.entries.map(e => [`${e.id}\t${e.heap}`, e]));
461const matrixIds = new Set(allCells().map(c => c.id));
462for (const entry of ratchet.entries) {
463 if (!matrixIds.has(entry.id)) {
464 process.stderr.write(`ratchet names ${entry.id}, which the matrix does not build\n`);
465 process.exit(1);
466 }
467}
469const fixtureRoot = join(outDir, 'fixtures');
470process.stderr.write(`sweep: ${selected.length} cells × 2 heaps, fixtures ${fixtureRoot}\n`);
471const oracle = await ensureFixtures(fixtureRoot, selected);
472const server = await startGitServer(fixtureRoot);
473const shim = await startShim(server.url);
474/** @type {Awaited<ReturnType<typeof decorate>>[]} */
475const cells = [];
476for (const cell of selected) cells.push(await decorate(cell, oracle, { root: fixtureRoot, server: server.url, shim: shim.url }));
477if (WORKERD && cells.some(c => c.runtime === 'workerd')) {
478 const dir = join(fixtureRoot, 'wd');
479 const config = buildWorker({ port: 0, outDir: dir });
480 wdBuild = { dir, config: readFileSync(config, 'utf8') };
481}
482/** @type {Record<string, unknown>[]} */
483const results = [];
484let done = 0;
485const jobs = cells.flatMap(cell => (cell.runtime === 'workerd' ? ['workerd'] : ['default', '128']).map(heap => ({ cell, heap })));
486try {
487 await pool(jobs, PAR, async ({ cell, heap }) => {
488 const result = await runOne(cell, heap, { server: server.url, shim: shim.url }, fixtureRoot);
489 results.push(result);
490 done += 1;
491 process.stderr.write(`[${done}/${jobs.length}] ${heap} ${cell.id} ${result.status}${result.code ? ` ${result.code}` : ''} rss ${result.rssPeakMB ?? '-'}MB\n`);
492 });
493} finally {
494 await shim.close();
495 await server.close();
496}
498const failures = [];
499const stale = [];
500const ratcheted = [];
501for (const result of results) {
502 const key = `${result.id}\t${result.heap}`;
503 const entry = known.get(key);
504 const view = viewOf(result);
505 const matched = covers(view, result.expect);
506 if (result.oracle) failures.push(result);
507 else if (matched && entry) stale.push(result);
508 else if (!matched && entry && covers(view, entry.actual)) ratcheted.push(result);
509 else if (!matched) failures.push(result);
510}
511for (const entry of ratchet.entries) {
512 if (!results.some(r => r.id === entry.id && r.heap === entry.heap)) continue;
513 // executed above
514}
516mkdirSync(outDir, { recursive: true });
517const jsonl = results.map(r => JSON.stringify(r)).join('\n') + '\n';
518writeFileSync(join(outDir, 'results.jsonl'), jsonl);
519writeFileSync(join(outDir, 'report.md'), markdown(results, failures, stale));
520process.stderr.write(`report ${join(outDir, 'report.md')}\n`);
521process.stderr.write(`ratchet held ${ratcheted.length}, divergences ${failures.length}, stale ${stale.length}\n`);
522if (failures.length || stale.length) process.exit(1);
Diagnostics4 git calls · 40 ms
CallStatusSizeTime040 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2008.6 KiB · hit35 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=33 git.response.bytes=8768 git.response.cache=HIT git.response.duration_ms=35 git.fetch.wants=bafb02af206241d96860115186dcc32b97628f7c git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=8768 git.pkt.payload.max=8747 git.sideband.band1.bytes=8746 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e30efb9e5e8a31
- Request
- 25781fce…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format