GitHubivarvong

ply

main
ply6 git calls342 ms31 KiB4 cachedprotocol v2
675 lines39 KiBRaw
1// Mutation gate. A full run on main; on a pull request, the mutants lying wholly inside the src lines the PR changed,
2// plus every line the gate judges strictly (gatedRanges: each parser function, each bounds file), whatever the PR
3// changed; the full run on every push to main catches the rest. Surviving or uncovered mutants inside a parser or a
4// bounds check fail the build. An equivalent mutant is allowed only as
5// `// Stryker disable next-line <Mutator>: <reason>` on that line — never `Stryker disable all`, and never a parser file
6// dropped from stryker.conf.json.
7//
8// node scripts/mutate.mjs run and judge in one process (local, or unsharded CI)
9// node scripts/mutate.mjs --shard i/N run shard i of N; write reports/mutation/shard-i.json; no verdict
10// node scripts/mutate.mjs --merge DIR --shards N
11// merge the N shard reports found under DIR and judge them
12// node scripts/mutate.mjs --plan N print the N-way partition (units, planned seconds and minutes, mutant
13// counts) and exit
14// node scripts/mutate.mjs --budget fail unless the full run's heaviest planned shard fits BUDGET_SHARE of
15// mutate.yml's timeout-minutes (CI: job lint + fuzz)
16// node scripts/mutate.mjs --check judge an existing reports/mutation/mutation.json
17// node scripts/mutate.mjs --weights [REPORT...] rewrite scripts/mutate-weights.json from full-run reports
18// node scripts/mutate.mjs --weights-pr REPORT... rewrite scripts/mutate-weights-pr.json from pull request shard
19// reports, newest first (a pull request plans from this table)
20//
21// The rules live only in this file. Every mode judges with the same summarize() and the same
22// verdict(); a shard only runs Stryker. The merge recomputes the partition from the commit and
23// refuses a missing, extra, duplicated or partial shard, so a lost artifact cannot pass the gate.
24import { spawn, spawnSync } from 'node:child_process';
25import { readFileSync, readdirSync, existsSync, mkdirSync, writeFileSync, statSync, appendFileSync } from 'node:fs';
26import { join } from 'node:path';
28const REPORT = 'reports/mutation/mutation.json';
29// A converted decoder keeps its gate: `xResult` is the Result core of the throwing wrapper `x` (docs/STACK.md).
30const PARSER = /^(parse[A-Z]\w*|readObjHeader|readOfsDelta|readUint32BE|readVarint|applyDelta|inflate\w*|huffBuild|adler32|pktStep|pktFinish|pktHeader|spStep|spFinish|v2Step|parseSideband|sidebandStep|member[A-Z]\w*|take|assertFetched|splitHeaders|checkIdent)(Result)?$/;
31// Either extension: a file keeps its gate when it converts to TypeScript. The pull codecs (#172) are decoders in every
32// function, their step helpers included: the whole file is gated.
33const BOUNDS_FILES = new Set(['src/bytes.js', 'src/format.js', 'src/bytes.ts', 'src/format.ts', 'src/inflatestream.ts', 'src/packstream.ts']);
34const SRC = /\.(?:js|ts)$/;
35// A test file that exits non-zero without reporting a failed test (a mutant that throws while its module loads, or one
36// that trips Stryker's hit limit, an infinite loop, there) is a RuntimeError to the tap runner. The test failed: the
37// mutant is detected, as it was under the command runner, where any non-zero exit was Killed.
38const DETECTED = new Set(['Killed', 'Timeout', 'RuntimeError']);
40function commentProblems() {
41 const bad = [];
42 for (const file of readdirSync('src').filter(f => SRC.test(f))) {
43 readFileSync(join('src', file), 'utf8').split('\n').forEach((line, i) => {
44 if (/Stryker disable all\b/.test(line)) bad.push(`${file}:${i + 1} uses a blanket Stryker disable`);
45 if (/Stryker disable next-line/.test(line)) {
46 const reason = line.slice(line.indexOf(':') + 1).trim();
47 const named = /Stryker disable next-line\s+\S[^:]*:/.test(line);
48 if (!named || reason.length < 8) bad.push(`${file}:${i + 1} equivalent mutant needs "Stryker disable next-line <Mutator>: <reason>"`);
49 }
50 });
51 }
52 const cfg = readFileSync('stryker.conf.json', 'utf8');
53 if (/!\s*src\/(objects|bytes|format|pack|packresolve|pktline|inflate|fetch|lsrefs)\.(?:js|ts)/.test(cfg)) {
54 bad.push('stryker.conf.json excludes a parser or bounds file');
55 }
56 return bad;
59function functions(file) {
60 const lines = readFileSync(file, 'utf8').split('\n');
61 const fns = [];
62 for (let i = 0; i < lines.length; i++) {
63 // A top-level const starts a new owner. Otherwise pktAttrs, sitting under pktFinish, would be
64 // scored as a parser survivor.
65 const m = lines[i].match(/^(?:export )?(?:async )?function (\w+)/)
66 || lines[i].match(/^(?:export )?const (\w+)\s*[:=]/);
67 if (m) fns.push({ name: m[1], line: i + 1 });
68 }
69 return fns;
72function owner(fns, line) {
73 let name = '<module>';
74 for (const fn of fns) if (fn.line <= line) name = fn.name;
75 return name;
78// The mutants summarize() judges strictly, as line ranges: file -> [[from, to], ...] (1-based, inclusive). A bounds
79// file is the whole file; elsewhere a range is a PARSER-named owner from its declaration line up to the line before the
80// next owner's (owner() gives a mutant to the last declaration at or above its first line, so these are exactly the
81// lines a strict mutant can start on). A pull request mutates them all, changed or not: a PR that only moves a parser,
82// or changes a test it relies on, changes no line of it, and its survivors would first show on main (41976594897: five
83// in parseHttpOptions and parseLsRefLine after #198 moved them). Same functions(), owner rule, PARSER and BOUNDS_FILES
84// as the verdict, so the two cannot drift.
85function gatedRanges(files) {
86 const out = new Map();
87 for (const file of files) {
88 const last = readFileSync(file, 'utf8').split('\n').length;
89 if (BOUNDS_FILES.has(file)) { out.set(file, [[1, last]]); continue; }
90 const fns = functions(file);
91 const rs = fns.flatMap((fn, k) => (PARSER.test(fn.name) ? [[fn.line, (fns[k + 1]?.line ?? last + 1) - 1]] : []));
92 if (rs.length) out.set(file, rs);
93 }
94 return out;
97// Sorted, overlapping or adjacent ranges joined: each mutant lies inside at most one, so the plan counts it once.
98function mergeRanges(rs) {
99 const merged = [];
100 for (const r of [...rs].sort((x, y) => x[0] - y[0])) {
101 const last = merged.at(-1);
102 if (last && r[0] <= last[1] + 1) last[1] = Math.max(last[1], r[1]); else merged.push([...r]);
103 }
104 return merged;
107// On a pull request, the src lines it changed: file -> [[from, to], ...] (1-based, inclusive, the PR's side; a pure
108// deletion borders the lines on either side of it). null on a full run (main, nightly, local, or MUTATE_FULL=1).
109function changedRanges() {
110 if (process.env.GITHUB_EVENT_NAME !== 'pull_request' || process.env.MUTATE_FULL === '1') return null;
111 const base = process.env.MUTATE_BASE || 'origin/main';
112 // --diff-filter=d: a file the PR deletes has nothing left to mutate.
113 const diff = spawnSync('git', ['diff', '-U0', '--no-color', '--diff-filter=d', `${base}...HEAD`, '--', 'src'], { encoding: 'utf8', maxBuffer: 1 << 28 });
114 if (diff.status !== 0) {
115 console.error(diff.stderr || diff.stdout);
116 process.exit(diff.status || 1);
117 }
118 const ranges = new Map();
119 let file = null;
120 for (const line of diff.stdout.split('\n')) {
121 if (line.startsWith('+++ ')) { const f = line.slice(4).replace(/^b\//, ''); file = f.startsWith('src/') && SRC.test(f) ? f : null; continue; }
122 const h = file && /^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/.exec(line);
123 if (!h) continue;
124 const from = Number(h[1]), len = h[2] === undefined ? 1 : Number(h[2]);
125 const r = len === 0 ? [Math.max(1, from), from + 1] : [from, from + len - 1];
126 if (!ranges.has(file)) ranges.set(file, []);
127 ranges.get(file).push(r);
128 }
129 for (const [f, rs] of ranges) ranges.set(f, mergeRanges(rs));
130 return ranges;
133// What a pull request mutates: the lines it changed and every strictly gated line (gatedRanges), merged per file.
134// null on a full run.
135function prRanges() {
136 const changed = changedRanges();
137 if (!changed) return null;
138 const out = new Map(changed);
139 for (const [f, rs] of gatedRanges(allSrc())) out.set(f, mergeRanges([...(out.get(f) ?? []), ...rs]));
140 return out;
143// Stryker applies `thresholds.break` itself, to whatever it mutated. On a pull request that is the changed and gated
144// lines only, so the full-library break would fail any PR touching a file the quick suites do not reach (read.js,
145// checkout.js: 0%).
146// The PR run gets the same config with the break off; the parser/bounds rule below is its gate. A shard is a subset
147// too, so it gets the same config: the break is applied by verdict() over the merged report of a full run.
148function noBreakConfig(path, extra = {}) {
149 const cfg = JSON.parse(readFileSync('stryker.conf.json', 'utf8'));
150 mkdirSync('reports', { recursive: true });
151 writeFileSync(path, JSON.stringify({ ...cfg, $schema: undefined, thresholds: { ...cfg.thresholds, break: null }, ...extra }, null, 2));
152 return path;
155function runStryker(files) {
156 const args = ['stryker', 'run'];
157 if (files) args.push(noBreakConfig('reports/stryker.pr.conf.json'), '--mutate', files.join(','));
158 const r = spawnSync('npx', args, { stdio: 'inherit' });
159 if (r.status !== 0) process.exit(r.status ?? 1);
162// Mutants per file, from Stryker's own instrumenter with the config's mutator options: the same mutants a run would
163// test, counted without running a test (about half a second for all of src/). The count is an upper bound on the cost:
164// with coverageAnalysis "perTest" a mutant no quick test reaches costs nothing (NoCoverage), and a covered one runs only
165// the test files that reach it.
166async function mutantCounts(files) {
167 const { Instrumenter } = await import('@stryker-mutator/instrumenter');
168 const quiet = () => false;
169 const noop = () => {};
170 const logger = { isTraceEnabled: quiet, isDebugEnabled: quiet, isInfoEnabled: quiet, isWarnEnabled: quiet, isErrorEnabled: quiet, isFatalEnabled: quiet, trace: noop, debug: noop, info: noop, warn: noop, error: noop, fatal: noop };
171 const mutator = JSON.parse(readFileSync('stryker.conf.json', 'utf8')).mutator ?? {};
172 const counts = new Map();
173 for (const file of files) {
174 const { mutants } = await new Instrumenter(logger).instrument([{ name: file, content: readFileSync(file, 'utf8'), mutate: true }], { plugins: null, excludedMutations: [], ...mutator, ignorers: [] });
175 // The instrumenter's lines are 0-based; Stryker's --mutate ranges and its report are 1-based.
176 const spans = mutants.map(m => [m.location.start.line + 1, m.location.end.line + 1]);
177 counts.set(file, { total: mutants.length, active: mutants.filter(m => !m.ignoreReason).length, spans, lines: readFileSync(file, 'utf8').split('\n').length });
178 }
179 return counts;
182function allSrc() {
183 // The config's mutate globs are src/**/*.{js,ts}; src/ has no subdirectories today. Refuse to guess if that changes.
184 const cfg = JSON.parse(readFileSync('stryker.conf.json', 'utf8'));
185 if (JSON.stringify(cfg.mutate) !== JSON.stringify(['src/**/*.js', 'src/**/*.ts'])) throw new Error('mutate: sharding assumes stryker.conf.json mutate is ["src/**/*.js", "src/**/*.ts"]');
186 const entries = readdirSync('src');
187 if (entries.some(e => statSync(join('src', e)).isDirectory())) throw new Error('mutate: sharding assumes src/ has no subdirectories');
188 return entries.filter(f => SRC.test(f)).map(f => `src/${f}`).sort();
191// Longest-processing-time first: heaviest unit to the lightest shard. Deterministic (ties by name, then by shard index),
192// so every shard and the gate compute the same plan from the same commit. A unit is a whole file on a full run; on a
193// pull request, a range of lines the PR changed or the gate judges strictly (prRanges; Stryker's --mutate file:a-b
194// mutates the mutants lying wholly inside it). A unit costing more than a shard's share is cut into ranges at lines no
195// mutant inside it spans past (push.js runs the push-outcome space for every mutant it reaches); a function whose body
196// is one block mutant cannot be cut, so the heaviest gated parser (parseTree) bounds a pull request's shard time. The
197// plan records how many mutants each file's units hold; the merge checks the reports against that count.
198async function plan(n) {
199 const ranges = prRanges();
200 const files = (ranges ? [...ranges.keys()] : allSrc()).filter(f => existsSync(f)).sort();
201 const counts = await mutantCounts(files);
202 const cost = costModel(ranges !== null);
203 const expected = new Map();
204 const costed = [];
205 for (const f of files) {
206 const c = counts.get(f);
207 const byLine = cost(f, c);
208 for (const [from, to] of ranges?.get(f) ?? [[1, c.lines]]) {
209 const inside = c.spans.filter(([a, b]) => a >= from && b <= to);
210 if (!inside.length) continue;
211 expected.set(f, (expected.get(f) ?? 0) + inside.length);
212 let weight = 0;
213 for (let l = from; l <= to; l++) weight += byLine.get(l) ?? 0;
214 costed.push({ file: f, from, to, whole: !ranges, inside, byLine, weight });
215 }
216 }
217 const share = costed.reduce((t, u) => t + u.weight, 0) / n;
218 const units = costed.flatMap(u => split(u, Math.ceil(u.weight / (share / CUTS_PER_SHARE) - 1e-9)))
219 .sort((a, b) => b.weight - a.weight || a.unit.localeCompare(b.unit));
220 const shards = Array.from({ length: n }, () => ({ files: [], weight: 0 }));
221 for (const { unit, weight } of units) {
222 let best = 0;
223 for (let i = 1; i < n; i++) if (shards[i].weight < shards[best].weight) best = i;
224 shards[best].files.push(unit);
225 shards[best].weight += weight;
226 }
227 for (const s of shards) s.files.sort();
228 return { full: ranges === null, files, expected, shards };
231// A unit is cut into pieces of at most a third of a shard's share, not a whole share: longest-processing-time packing
232// of share-sized pieces left the heaviest of 36 shards at 34.0 planned minutes against a 24.6 mean (#313); thirds bring
233// it to 25.1. Only a function costing more than a third of a share, whose body is one block mutant, stays whole.
234const CUTS_PER_SHARE = 3;
236// A unit as k ranges of about equal measured cost, each ending on a line none of its mutants spans past. k <= 1, or a
237// unit with no such line near a cut, stays whole. A slice no mutant lies wholly inside is dropped: line cost counts
238// every mutant that *starts* on the line, including one that ends outside the changed range (Stryker's --mutate
239// range keeps only a mutant wholly inside it). Such a slice mutates nothing, the report has no file entry, and the
240// merge rejects it (http.ts:106 and http.ts:228 on #133).
241function split({ file, from, to, whole, inside, byLine, weight }, k) {
242 const label = (a, b) => (whole && a === from && b === to ? file : `${file}:${a}-${b}`);
243 const held = (a, b) => inside.some(([s, e]) => s >= a && e <= b);
244 if (k <= 1) return [{ unit: label(from, to), weight }];
245 const crossed = new Set();
246 for (const [a, b] of inside) for (let l = a; l < b; l++) crossed.add(l);
247 const upTo = new Map([[from - 1, 0]]); // cost of the unit's lines up to and including l
248 for (let l = from; l <= to; l++) upTo.set(l, upTo.get(l - 1) + (byLine.get(l) ?? 0));
249 const cuts = [];
250 for (let j = 1; j < k; j++) {
251 const want = (j * weight) / k, prev = upTo.get(cuts.at(-1) ?? from - 1);
252 let best = null;
253 for (let b = (cuts.at(-1) ?? from - 1) + 1; b < to; b++) {
254 if (crossed.has(b) || upTo.get(b) <= prev || upTo.get(b) >= weight) continue;
255 if (best === null || Math.abs(upTo.get(b) - want) < Math.abs(upTo.get(best) - want)) best = b;
256 }
257 if (best !== null) cuts.push(best);
258 }
259 const bounds = [from - 1, ...cuts, to];
260 return bounds.slice(1).flatMap((b, i) => {
261 const start = bounds[i] + 1;
262 if (!held(start, b)) return [];
263 return [{ unit: label(start, b), weight: upTo.get(b) - upTo.get(bounds[i]) }];
264 });
267const unitFile = unit => unit.replace(/:\d+-\d+$/, '');
269// With per-test coverage a mutant's cost is the test files that reach it, not 1: linemerge.js (13% of the mutants) is all
270// NoCoverage and free, while a push.js mutant can run the push-outcome space. scripts/mutate-weights.json holds measured
271// seconds of test time per file and per line (by the line a mutant starts on), from full-run reports (refresh with
272// --weights REPORT...; the first report holding a file wins). A line with no entry costs the file's mean seconds per
273// mutant for each mutant starting there; a file with no entry, the median known rate. Committed, so every shard and the
274// gate compute the same plan.
275// A pull request has its own table, scripts/mutate-weights-pr.json, measured from pull request shard reports
276// (--weights-pr REPORT...): the same line costs a different amount in the two modes, so one table cannot plan both.
277// On #243/#244 the PR runs cost attributes.ts:235-300 at 5,943-6,604 s (the full-run table said 413 s, so the
278// planner never cut it and one shard ran 36-51 min) and objects.ts:279-347 at 11 s (the full run measured 3,688 s).
279// A pull request plans from its table, falling back per line to the full-run table, then to a rate.
280//
281// Seconds are seconds on a reference runner. GitHub's ubuntu-latest runners are not one machine: the dry run of the same
282// 412 mutation tests took 28.6-94.0 s net across the 20 shards of 43232620788 (26.9-58.3 s on 43177287797), and a
283// mutant's duration scales with it. push.ts:293-1028 was measured on two 28 s runners (3502eef) and ran on two 48-50 s
284// ones (b65917f): 9,322 s planned, 19,065 s measured, an 84-minute shard against the 90-minute limit (#313). So each
285// shard records its runner's dry-run time (Stryker's "net" ms, report.shard.dryRunNetMs), and --weights scales every
286// duration to REF_DRY_MS. REF_DRY_MS is a slow runner (11 of those 20 ran 47-57 s), so a typical runner finishes under
287// its plan and a 57 s one within about 15% of it.
288const REF_DRY_MS = 50_000;
289// A shard's wall minutes from its seconds of mutant time: setup (checkout, npm ci, instrumenting, the dry run) plus
290// the mutants over Stryker's effective parallelism. Least squares over the 20 shards of 43273467698 (f47bb75): job
291// minutes against each shard's measured mutant seconds on its own runner, a timed-out mutant counted at its duration
292// plus TIMEOUT_RESTART_S. Within 3.4 min on every shard (the timeout-blind fit of 43232620788, 3.84 + s / 3.88 / 60,
293// was 5.6 min short on shard 16 and 4.4 on shard 19, the shards holding the most timeouts).
294const WALL_SETUP_MIN = 2.15;
295const WALL_PARALLEL = 3.76;
296// A timed-out mutant costs more than its recorded duration: Stryker kills the hung test process and starts a new one
297// before its next mutant. The same fit puts that at 10.0 s of runner time per Timeout (85 timeouts on shard 19,
298// 67 min 41 s against 63.3 planned without it). --weights adds it to every Timeout's line.
299const TIMEOUT_RESTART_S = 10;
300const plannedMinutes = seconds => WALL_SETUP_MIN + seconds / WALL_PARALLEL / 60;
301// --budget: the rule a full run's plan must meet. Its heaviest shard, planned on the reference runner, must still finish
302// within BUDGET_SHARE of mutate.yml's timeout-minutes (60 of 90) on a runner RUNNER_WORST times slower:
303// plannedMinutes(max) * RUNNER_WORST <= BUDGET_SHARE * timeout-minutes
304// RUNNER_WORST is the slowest runner seen: shard 9 of 43273467698 dry-ran in 99.3 s (1.99x the reference) and took
305// 79 min 17 s; 94.0 s on 43232620788. The whole planned time scales (setup included), which over-counts npm ci and
306// checkout: the margin is on the safe side. The remaining third of the limit is for weights gone stale.
307// Since then one runner has been far slower than RUNNER_WORST: shard 14 of 43639075968 (a78ed24) dry-ran in
308// 242.2 s, 4.84x the reference; the other 35 shards ran 32.7-91.4 s. It held 25.5 planned minutes, one of 36 near-equal
309// shards, and hit the 90-minute limit at 412 of 499 mutants. No plan within this rule absorbs such a runner (at 4.84x a
310// shard must plan under 18.6 min); the rule is unchanged and a rerun of that shard is the remedy.
311// An all-NoCoverage file costs about 0 s: the ply/probes files (src/probe-*.ts, 1,727 mutants, none reached by the
312// mutation suite) measured 0 s on that run, against 7,533 s planned at the median rate before they had weights.
313const RUNNER_WORST = 2;
314const BUDGET_SHARE = 2 / 3;
315const WEIGHTS = 'scripts/mutate-weights.json';
316const WEIGHTS_PR = 'scripts/mutate-weights-pr.json';
317const readWeights = path => (existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : {});
318function costModel(pr) {
319 const full = readWeights(WEIGHTS);
320 const tables = pr ? [readWeights(WEIGHTS_PR), full] : [full];
321 const rates = Object.values(full).filter(w => w.mutants > 0).map(w => w.seconds / w.mutants).sort((a, b) => a - b);
322 const median = rates.length ? rates[rates.length >> 1] : 1;
323 // Seconds per line for this file's current mutants (spans from the instrumenter).
324 return (file, { spans }) => {
325 const ws = tables.map(t => t[file]);
326 const known = ws.find(w => w?.mutants);
327 const rate = known ? known.seconds / known.mutants : median;
328 const startsAt = new Map();
329 for (const [a] of spans) startsAt.set(a, (startsAt.get(a) ?? 0) + 1);
330 const byLine = new Map();
331 for (const [line, n] of startsAt) byLine.set(line, ws.map(w => w?.byLine?.[line]).find(s => s !== undefined) ?? n * rate);
332 return byLine;
333 };
336// Measured seconds per file and per line (by the line a mutant starts on). Merged by line: the first report holding a
337// line wins it, so a file a full run split across shards, or that several pull requests each covered in part, keeps
338// every measured line (holding only a file's first report dropped the rest: inflatestream.ts on main's 16-way plan).
339// seconds and mutants are the totals over the kept lines.
340// Each duration is scaled to the reference runner by the dry-run time of the runner that measured it (a shard report's
341// shard.dryRunNetMs, or a merged report's per-mutant runnerDryMs); a report without one is refused, not guessed. A
342// Timeout also carries TIMEOUT_RESTART_S: shards 16 and 19 of 43273467698 held 49 and 85 timeouts and ran 5.6 and 4.4
343// min past a plan that counted only durations.
344function writeWeights(paths, target) {
345 const out = {};
346 const unscaled = [];
347 for (const path of paths) {
348 const report = JSON.parse(readFileSync(path, 'utf8'));
349 const files = report.files ?? {};
350 for (const [file, info] of Object.entries(files)) {
351 const lines = new Map();
352 for (const m of info.mutants ?? []) {
353 const dry = m.runnerDryMs ?? report.shard?.dryRunNetMs ?? null;
354 if (!(dry > 0)) { unscaled.push(`${path}: ${file}:${m.location.start.line}`); continue; }
355 const l = m.location.start.line;
356 const e = lines.get(l) ?? { s: 0, n: 0 };
357 // a Timeout at its real cost: its duration and the test process restart after it (TIMEOUT_RESTART_S)
358 const runnerSeconds = (m.duration ?? 0) / 1000 + (m.status === 'Timeout' ? TIMEOUT_RESTART_S : 0);
359 e.s += runnerSeconds * (REF_DRY_MS / dry);
360 if (m.status !== 'Ignored') e.n += 1;
361 lines.set(l, e);
362 }
363 const w = (out[file] ??= { seconds: 0, mutants: 0, byLine: {} });
364 for (const [l, { s, n }] of lines) {
365 if (w.byLine[l] !== undefined) continue;
366 w.byLine[l] = Math.round(s * 10) / 10;
367 w.seconds += s;
368 w.mutants += n;
369 }
370 }
371 }
372 if (unscaled.length) {
373 console.error(`mutate: ${unscaled.length} mutant(s) without their runner's dry-run time, so their seconds cannot be scaled to the reference runner (first: ${unscaled[0]}); refresh weights from reports a shard of this version wrote`);
374 process.exit(2);
375 }
376 for (const w of Object.values(out)) w.seconds = Math.round(w.seconds * 10) / 10;
377 const sorted = Object.fromEntries(Object.keys(out).sort().map(k => [k, out[k]]));
378 writeFileSync(target, JSON.stringify(sorted) + '\n');
379 console.log(`mutate: wrote ${target} from ${paths.length} report(s) (${Object.keys(out).length} files)`);
382function parseShard(arg) {
383 const m = /^(\d+)\/(\d+)$/.exec(arg ?? '');
384 const i = m ? Number(m[1]) : NaN, n = m ? Number(m[2]) : NaN;
385 if (!(n >= 1 && i >= 1 && i <= n)) {
386 console.error(`mutate: --shard wants i/N with 1 <= i <= N, got ${arg}`);
387 process.exit(2);
388 }
389 return { i, n };
392const shardReport = i => `reports/mutation/shard-${i}.json`;
394async function runShard(i, n) {
395 const p = await plan(n);
396 const mine = p.shards[i - 1];
397 console.log(`mutate: shard ${i}/${n} of ${p.full ? 'full src/' : `${p.files.length} file(s): changed and gated lines`}; planned seconds per shard ${p.shards.map(s => Math.round(s.weight)).join(', ')}`);
398 console.log(`mutate: planned minutes per shard on the reference runner ${p.shards.map(s => plannedMinutes(s.weight).toFixed(1)).join(', ')}`);
399 mkdirSync('reports/mutation', { recursive: true });
400 if (mine.files.length === 0) {
401 writeFileSync(shardReport(i), JSON.stringify({ shard: { i, n, files: [] }, files: {} }));
402 console.log(`mutate: shard ${i}/${n} has no files; wrote an empty report`);
403 return;
404 }
405 console.log(`${mine.files.join('\n')}\n(${Math.round(mine.weight)} planned seconds)`);
406 // Own report path and temp dir (under .stryker-tmp, which ignorePatterns already excludes), so shards can run side
407 // by side in one checkout.
408 const cfg = noBreakConfig(`reports/stryker.shard-${i}.conf.json`, {
409 tempDirName: `.stryker-tmp/shard-${i}`,
410 jsonReporter: { fileName: shardReport(i) },
411 });
412 const dryRunNetMs = await strykerTimed(['stryker', 'run', cfg, '--mutate', mine.files.join(',')]);
413 if (dryRunNetMs === null) console.warn(`mutate: shard ${i}/${n}: no dry-run time in Stryker's output; this report cannot refresh weights`);
414 else console.log(`mutate: shard ${i}/${n}: runner dry run ${(dryRunNetMs / 1000).toFixed(1)} s net (reference ${REF_DRY_MS / 1000} s)`);
415 runnerSummary(i, n, mine.weight, dryRunNetMs);
416 const report = JSON.parse(readFileSync(shardReport(i), 'utf8'));
417 report.shard = { i, n, files: mine.files, dryRunNetMs };
418 writeFileSync(shardReport(i), JSON.stringify(report));
421// One line in the job summary: this runner's speed against the reference (its dry run over REF_DRY_MS) and the shard's
422// planned minutes at the reference and at that speed, so a slow runner shows on the run page, not only in the log
423// (the factor --budget's RUNNER_WORST must stay above).
424function runnerSummary(i, n, weight, dryRunNetMs) {
425 const path = process.env.GITHUB_STEP_SUMMARY;
426 if (!path) return;
427 const planned = plannedMinutes(weight);
428 const line = dryRunNetMs === null
429 ? `mutate shard ${i}/${n}: no dry-run time in Stryker's output; planned ${planned.toFixed(1)} min on the reference runner`
430 : `mutate shard ${i}/${n}: runner factor ${(dryRunNetMs / REF_DRY_MS).toFixed(2)}x (dry run ${(dryRunNetMs / 1000).toFixed(1)} s, reference ${REF_DRY_MS / 1000} s, budget ${RUNNER_WORST}x); planned ${planned.toFixed(1)} min at the reference, ${(planned * dryRunNetMs / REF_DRY_MS).toFixed(1)} at this runner`;
431 appendFileSync(path, `${line}\n`);
434// Stryker run with its output passed through as it comes, returning the runner's speed: the net ms of Stryker's dry run
435// ("Initial test run succeeded. Ran 412 tests in 58 seconds (net 49946.1 ms, overhead 8473.9 ms)"), or null when the
436// line is not there. A non-zero exit ends the shard with Stryker's status.
437const DRY_RUN = /Initial test run succeeded\. Ran \d+ tests? in [^(]*\(net ([0-9.]+) ms/;
438function strykerTimed(args) {
439 return new Promise(resolve => {
440 const child = spawn('npx', args, { stdio: ['inherit', 'pipe', 'pipe'] });
441 let dry = null;
442 let tail = '';
443 const watch = sink => chunk => {
444 sink.write(chunk);
445 if (dry !== null) return;
446 tail = (tail + chunk.toString('utf8')).slice(-8192);
447 const m = DRY_RUN.exec(tail);
448 if (m) dry = Number(m[1]);
449 };
450 child.stdout.on('data', watch(process.stdout));
451 child.stderr.on('data', watch(process.stderr));
452 child.on('close', code => {
453 if (code !== 0) process.exit(code ?? 1);
454 resolve(dry);
455 });
456 });
459// The full run's plan against the job: the heaviest planned shard's minutes on the reference runner (plannedMinutes),
460// times RUNNER_WORST, at most BUDGET_SHARE of mutate-shard's timeout-minutes, with the shard count and limit read from
461// mutate.yml itself. Cheap (the instrumenter, no test), so CI runs it on every push: a weights refresh, a new src file or
462// a shard count that would plan a shard past the bar on the slowest runner fails here, not 80 minutes into a mutation
463// run (#313: 79 min 17 s on a 1.99x runner against a plan that allowed 54 at the reference).
464async function budget() {
465 const yml = readFileSync('.github/workflows/mutate.yml', 'utf8');
466 // the job's own lines, each after a newline: from ` mutate-shard:` up to the next two-space key
467 const job = `\n${/\n {2}mutate-shard:\n([\s\S]*?)\n {2}[\w-]+:\n/.exec(yml)?.[1] ?? ''}`;
468 const limit = Number(/\n {4}timeout-minutes: (\d+)\n/.exec(job)?.[1]);
469 const matrix = /\n {8}shard: \[([\d, ]+)\]/.exec(job)?.[1];
470 const n = matrix ? matrix.split(',').length : NaN;
471 if (!(limit > 0) || !(n >= 1)) {
472 console.error('mutate: --budget cannot read the mutate-shard job\'s timeout-minutes and shard matrix from .github/workflows/mutate.yml');
473 process.exit(2);
474 }
475 process.env.MUTATE_FULL = '1';
476 const p = await plan(n);
477 const minutes = p.shards.map(s => plannedMinutes(s.weight));
478 const max = Math.max(...minutes);
479 const worst = max * RUNNER_WORST;
480 const cap = BUDGET_SHARE * limit;
481 const heaviest = p.shards[minutes.indexOf(max)];
482 console.log(`mutate: full run in ${n} shards; planned minutes on the reference runner (dry run ${REF_DRY_MS / 1000} s) max ${max.toFixed(1)}, mean ${(minutes.reduce((t, m) => t + m, 0) / n).toFixed(1)}; on a ${RUNNER_WORST}x runner max ${worst.toFixed(1)}; cap ${cap.toFixed(1)} (${Math.round(BUDGET_SHARE * 100)}% of timeout-minutes ${limit})`);
483 if (worst > cap + 1e-9) {
484 console.error(`mutate: shard ${minutes.indexOf(max) + 1}/${n} plans ${max.toFixed(1)} min, ${worst.toFixed(1)} on a ${RUNNER_WORST}x runner, over ${cap.toFixed(1)}: ${heaviest.files.join(', ')}. Add shards (every ${n} of mutate.yml), or refresh weights with --weights`);
485 process.exit(1);
486 }
489function findReports(dir) {
490 const out = [];
491 for (const e of readdirSync(dir)) {
492 const p = join(dir, e);
493 if (statSync(p).isDirectory()) out.push(...findReports(p));
494 else if (/^shard-\d+\.json$/.test(e)) out.push(p);
495 }
496 return out;
499// Union of the shard reports, after checking that together they are exactly the plan: each shard once, each shard's
500// files exactly its planned files, each planned file in one report. Anything else fails the gate.
501async function mergeReports(dir, n) {
502 const p = await plan(n);
503 const bad = [];
504 const seen = new Map();
505 for (const path of findReports(dir)) {
506 const r = JSON.parse(readFileSync(path, 'utf8'));
507 const i = r.shard?.i;
508 if (r.shard?.n !== n || !(i >= 1 && i <= n)) { bad.push(`${path}: shard ${JSON.stringify(r.shard)} is not one of ${n}`); continue; }
509 if (seen.has(i)) { bad.push(`${path}: shard ${i} reported twice (also ${seen.get(i).path})`); continue; }
510 seen.set(i, { path, r });
511 }
512 const merged = { files: {} };
513 for (let i = 1; i <= n; i++) {
514 const want = p.shards[i - 1].files;
515 const got = seen.get(i);
516 if (!got) { bad.push(`shard ${i}/${n}: no report (planned ${want.length} file(s))`); continue; }
517 const { r } = got;
518 if (JSON.stringify(r.shard.files) !== JSON.stringify(want)) bad.push(`shard ${i}/${n}: ran ${JSON.stringify(r.shard.files)}, plan is ${JSON.stringify(want)}`);
519 const keys = Object.keys(r.files ?? {}).sort();
520 const wantFiles = [...new Set(want.map(unitFile))].sort();
521 if (JSON.stringify(keys) !== JSON.stringify(wantFiles)) bad.push(`shard ${i}/${n}: report covers ${JSON.stringify(keys)}, plan is ${JSON.stringify(wantFiles)}`);
522 // each mutant keeps its runner's dry-run time, so --weights can scale the merged report too
523 const runnerDryMs = r.shard.dryRunNetMs ?? null;
524 for (const [file, info] of Object.entries(r.files ?? {})) {
525 const mutants = (info.mutants ?? []).map(m => ({ ...m, runnerDryMs }));
526 if (merged.files[file]) merged.files[file] = { ...merged.files[file], mutants: [...merged.files[file].mutants, ...mutants] };
527 else merged.files[file] = { ...info, mutants };
528 }
529 for (const k of Object.keys(r)) if (k !== 'files' && k !== 'shard' && !(k in merged)) merged[k] = r[k];
530 }
531 // Every mutant of every planned file exactly once: a lost range, or a mutant no range held, is a short count.
532 for (const file of new Set(p.shards.flatMap(sh => sh.files.map(unitFile)))) {
533 const ms = merged.files[file]?.mutants ?? [];
534 const keys = new Set(ms.map(m => `${m.location.start.line}:${m.location.start.column}-${m.location.end.line}:${m.location.end.column} ${m.mutatorName} ${m.replacement}`));
535 if (ms.length !== p.expected.get(file) || keys.size !== ms.length) bad.push(`${file}: ${ms.length} mutant(s) reported (${keys.size} distinct), the plan's ranges hold ${p.expected.get(file)}`);
536 }
537 if (bad.length) {
538 console.error(`mutate: shard reports do not match the plan\n${bad.join('\n')}`);
539 process.exit(1);
540 }
541 console.log(`mutate: merged ${n} shard report(s), ${Object.keys(merged.files).length} file(s), every planned mutant once, of ${p.full ? 'full src/' : `the changed and gated lines of ${p.files.length} file(s)`}`);
542 mkdirSync('reports/mutation', { recursive: true });
543 writeFileSync(REPORT, JSON.stringify(merged));
544 return merged;
547function loadReport() {
548 if (!existsSync(REPORT)) {
549 console.error(`no mutation report at ${REPORT}`);
550 process.exit(1);
551 }
552 return JSON.parse(readFileSync(REPORT, 'utf8'));
555function summarize(report) {
556 const fileMap = report.files ?? {};
557 const rows = [];
558 const violations = [];
559 let detected = 0, survived = 0, ignored = 0;
560 for (const [file, info] of Object.entries(fileMap)) {
561 const mutants = info.mutants ?? [];
562 const fns = existsSync(file) ? functions(file) : [];
563 const counts = {};
564 for (const m of mutants) {
565 const status = m.status;
566 counts[status] = (counts[status] ?? 0) + 1;
567 if (DETECTED.has(status)) detected++;
568 else if (status === 'Ignored') ignored++;
569 else survived++;
570 const line = m.location?.start?.line ?? 0;
571 const fn = owner(fns, line);
572 const strict = BOUNDS_FILES.has(file) || PARSER.test(fn);
573 if (strict && (status === 'Survived' || status === 'NoCoverage')) {
574 violations.push(`${file}:${line} ${fn} ${m.mutatorName} ${status}${m.replacement ? ` → ${String(m.replacement).slice(0, 60)}` : ''}`);
575 }
576 }
577 const det = (counts.Killed ?? 0) + (counts.Timeout ?? 0) + (counts.RuntimeError ?? 0);
578 const sur = (counts.Survived ?? 0) + (counts.NoCoverage ?? 0);
579 const score = det + sur === 0 ? 100 : (100 * det) / (det + sur);
580 rows.push({ file, score, killed: counts.Killed ?? 0, timeout: counts.Timeout ?? 0, survived: counts.Survived ?? 0, noCoverage: counts.NoCoverage ?? 0, ignored: counts.Ignored ?? 0, total: mutants.length });
581 }
582 rows.sort((a, b) => a.file.localeCompare(b.file));
583 const score = detected + survived === 0 ? 100 : (100 * detected) / (detected + survived);
584 return { rows, violations, score, detected, survived, ignored };
587function verdict(report) {
588 const { rows, violations, score, detected, survived, ignored } = summarize(report);
589 const breakAt = JSON.parse(readFileSync('stryker.conf.json', 'utf8')).thresholds?.break;
590 // A pull request mutates only the files it changed. That subset's percentage is not
591 // comparable to the full-library break, so the threshold applies to a full run.
592 const fullRun = process.env.GITHUB_EVENT_NAME !== 'pull_request' || process.env.MUTATE_FULL === '1';
593 printRows(rows);
594 console.log(`\nscore ${score.toFixed(2)} (${detected} detected, ${survived} survived/uncovered, ${ignored} ignored)`);
595 if (fullRun && breakAt !== null && breakAt !== undefined && score + 1e-9 < breakAt) {
596 console.error(`mutation score ${score.toFixed(2)} is below the break threshold ${breakAt}`);
597 process.exit(1);
598 }
599 if (violations.length) {
600 console.error(`\n${violations.length} surviving mutant(s) in a parser or bounds check:`);
601 console.error(violations.join('\n'));
602 process.exit(1);
603 }
606function printRows(rows) {
607 console.log('\nper-file mutation score');
608 console.log('file score killed timeout survived uncovered ignored');
609 for (const r of rows) {
610 console.log(`${r.file.padEnd(28)} ${r.score.toFixed(1).padStart(5)} ${String(r.killed).padStart(6)} ${String(r.timeout).padStart(7)} ${String(r.survived).padStart(8)} ${String(r.noCoverage).padStart(9)} ${String(r.ignored).padStart(7)}`);
611 }
614function argValue(flag) {
615 const k = process.argv.indexOf(flag);
616 return k === -1 ? undefined : process.argv[k + 1];
619const problems = commentProblems();
620if (problems.length) {
621 console.error(problems.join('\n'));
622 process.exit(1);
625if (process.argv.includes('--weights-pr')) {
626 const k = process.argv.indexOf('--weights-pr');
627 const paths = process.argv.slice(k + 1).filter(a => !a.startsWith('--'));
628 if (!paths.length) {
629 console.error('mutate: --weights-pr wants pull request shard reports (newest first)');
630 process.exit(2);
631 }
632 writeWeights(paths, WEIGHTS_PR);
633} else if (process.argv.includes('--weights')) {
634 const k = process.argv.indexOf('--weights');
635 const paths = process.argv.slice(k + 1).filter(a => !a.startsWith('--'));
636 writeWeights(paths.length ? paths : [REPORT], WEIGHTS);
637} else if (process.argv.includes('--budget')) {
638 await budget();
639} else if (process.argv.includes('--plan')) {
640 const n = Number(argValue('--plan'));
641 if (!(n >= 1)) {
642 console.error('mutate: --plan wants N >= 1');
643 process.exit(2);
644 }
645 const p = await plan(n);
646 const mutants = [...p.expected.values()].reduce((t, k) => t + k, 0);
647 const shards = p.shards.map(s => ({ ...s, minutes: Math.round(plannedMinutes(s.weight) * 10) / 10 }));
648 console.log(JSON.stringify({ full: p.full, mutants, expected: Object.fromEntries(p.expected), shards }, null, 2));
649} else if (process.argv.includes('--shard')) {
650 const { i, n } = parseShard(argValue('--shard'));
651 await runShard(i, n);
652 // Per-shard table for the log only. The verdict is the merge's.
653 printRows(summarize(JSON.parse(readFileSync(shardReport(i), 'utf8'))).rows);
654} else if (process.argv.includes('--merge')) {
655 const dir = argValue('--merge');
656 const n = Number(argValue('--shards'));
657 if (!dir || !existsSync(dir) || !(n >= 1)) {
658 console.error('mutate: --merge wants an existing directory and --shards N');
659 process.exit(2);
660 }
661 verdict(await mergeReports(dir, n));
662} else {
663 if (!process.argv.includes('--check')) {
664 const p = await plan(1);
665 const units = p.full ? null : p.shards[0].files;
666 if (units && units.length === 0) {
667 console.log('mutate: pull request changes no mutable src/ line and no line is gated; skipping');
668 process.exit(0);
669 }
670 if (units) console.log(`mutate: the lines the pull request changed, and every gated line\n${units.join('\n')}`);
671 else console.log('mutate: full src/');
672 runStryker(units);
673 }
674 verdict(loadReport());
Diagnostics6 git calls · 342 ms
CallStatusSizeTime0342 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=1
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit1 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=1
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:0200831 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=334
git.response.bytes=831
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=b9290e370bd091fd3272931655a175dd1a7af82d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=831
git.pkt.payload.max=804
git.sideband.band1.bytes=804
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=804
git.pack.trailer=81ddc1f73214935b4ee953ad84f959db955ce0d1
Open pack map
fetch20013 KiB334 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=13695
git.response.cache=MISS
git.response.duration_ms=334
git.fetch.wants=274d132dabacb026747ef15670dfb8813490295b
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=5
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=13695
git.pkt.payload.max=8192
git.sideband.band1.bytes=13663
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0f8472cfa332c
Request
65e34985…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format