main
1// scripts/mutate.mjs on a pull request mutates every line the gate judges strictly, not only the lines the PR changed.
2// #198 moved parseHttpOptions and parseLsRefLine without changing their text: no line of theirs was in the diff, the PR's
3// mutation run never reached them, and main's full run found five survivors there (run 41976594897). Here a PR that
4// only moves a helper past a parser leaves the parser's lines out of the diff; the plan still holds all of them, and
5// still leaves out an ungated function the PR did not change.
6import { test } from 'vitest';
7import assert from 'node:assert/strict';
8import { spawnSync } from 'node:child_process';
9import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
10import { tmpdir } from 'node:os';
11import { join } from 'node:path';
12import { fileURLToPath } from 'node:url';
14const ROOT = fileURLToPath(new URL('..', import.meta.url));
15const SCRIPT = join(ROOT, 'scripts/mutate.mjs');
17// parseFrame is gated (PARSER); helper and other are not. 40 statements, so git's diff moves the 3-line helper, not
18// the parser.
19const parser = [
20 'export function parseFrame(b: Uint8Array): number {',
21 ' let n = 0;',
22 ...Array.from({ length: 40 }, (_, i) => ` if (b[${i}] === ${i + 1}) n += ${i + 2};`),
23 ' return n;',
24 '}',
25];
26const helper = ['export function helper(x: number): number {', ' return x > 1 ? x * 2 : x - 3;', '}'];
27const other = ['export function other(x: number): number {', ' return x < 4 ? x + 5 : x * 6;', '}'];
28const lines = parts => parts.flat().join('\n') + '\n';
30function git(cwd, ...args) {
31 const r = spawnSync('git', args, { cwd, encoding: 'utf8' });
32 assert.equal(r.status, 0, `git ${args.join(' ')}: ${r.stderr}`);
33 return r.stdout.trim();
34}
36function prPlan(cwd, base) {
37 const env = { ...process.env, GITHUB_EVENT_NAME: 'pull_request', MUTATE_BASE: base };
38 delete env.MUTATE_FULL;
39 const r = spawnSync(process.execPath, [SCRIPT, '--plan', '4'], { cwd, env, encoding: 'utf8' });
40 assert.equal(r.status, 0, r.stderr);
41 return JSON.parse(r.stdout);
42}
44// [from, to] ranges of a file's units in a plan
45const unitRanges = (plan, file) => plan.shards.flatMap(s => s.files)
46 .filter(u => u === file || u.startsWith(`${file}:`))
47 .map(u => (u === file ? [1, Infinity] : u.slice(file.length + 1).split('-').map(Number)));
48const covers = (ranges, from, to) => ranges.some(([a, b]) => a <= from && b >= to);
50test('a pull request that only moves code past a parser still mutates every line of the parser', () => {
51 const dir = mkdtempSync(join(tmpdir(), 'ply-mutate-plan-'));
52 try {
53 mkdirSync(join(dir, 'src'));
54 copyFileSync(join(ROOT, 'stryker.conf.json'), join(dir, 'stryker.conf.json'));
55 git(dir, 'init', '-q', '-b', 'main');
56 git(dir, 'config', 'user.email', 't@example.com');
57 git(dir, 'config', 'user.name', 't');
58 writeFileSync(join(dir, 'src/a.ts'), lines([helper, parser, other]));
59 git(dir, 'add', '.');
60 git(dir, 'commit', '-q', '-m', 'base');
61 const base = git(dir, 'rev-parse', 'HEAD');
62 writeFileSync(join(dir, 'src/a.ts'), lines([parser, helper, other]));
63 git(dir, 'commit', '-q', '-am', 'move helper below the parser');
65 // the parser's lines on the PR side: 1..parser.length. The changed lines (a pure deletion borders the lines on
66 // either side of it, as in changedRanges) do not cover them: changed lines alone would not mutate the parser.
67 const from = 1, to = parser.length;
68 const hunks = git(dir, 'diff', '-U0', `${base}...HEAD`).split('\n').filter(l => l.startsWith('@@'));
69 const changed = hunks.map(h => /\+(\d+)(?:,(\d+))?/.exec(h)).map(m => {
70 const at = Number(m[1]), len = Number(m[2] ?? 1);
71 return len === 0 ? [Math.max(1, at), at + 1] : [at, at + len - 1];
72 });
73 assert.ok(!covers(changed, from, to), `the diff changes the parser: ${hunks.join(' ')}`);
74 assert.ok(changed.every(([a, b]) => b <= from || a > to), `the diff changes the parser: ${hunks.join(' ')}`);
76 const plan = prPlan(dir, base);
77 assert.equal(plan.full, false);
78 const ranges = unitRanges(plan, 'src/a.ts');
79 assert.ok(covers(ranges, from, to), `parseFrame ${from}-${to} not in the plan: ${JSON.stringify(ranges)}`);
80 // the moved helper is there too (changed lines), the untouched ungated function is not
81 const helperAt = parser.length + 1;
82 assert.ok(covers(ranges, helperAt, helperAt + helper.length - 1), JSON.stringify(ranges));
83 const otherAt = parser.length + helper.length + 1;
84 assert.ok(!ranges.some(([a, b]) => a <= otherAt + 1 && b >= otherAt + 1), `other() is in the plan: ${JSON.stringify(ranges)}`);
85 // each planned mutant once: the parser's 40 conditions alone are well over 40
86 assert.ok(plan.mutants > 40, String(plan.mutants));
87 } finally {
88 rmSync(dir, { recursive: true, force: true });
89 }
90}, 60_000);
92// #313: a shard of 43232620788 planned at 9,322 s ran 19,065 s (84 of 90 minutes), because its weights were measured on
93// a runner whose dry run took 28 s and it ran on one taking 50 s. --weights scales every duration to the reference
94// runner by the dry-run time of the runner that measured it, and refuses a report that does not say.
95const mutant = (line, ms) => ({ id: String(line), mutatorName: 'BooleanLiteral', replacement: 'true', status: 'Killed', duration: ms, location: { start: { line, column: 3 }, end: { line, column: 9 } } });
96const shardReport = (dryRunNetMs, mutants) => JSON.stringify({ shard: { i: 1, n: 1, files: ['src/a.ts'], dryRunNetMs }, files: { 'src/a.ts': { mutants } } });
98function scratch() {
99 const dir = mkdtempSync(join(tmpdir(), 'ply-mutate-weights-'));
100 mkdirSync(join(dir, 'src'));
101 mkdirSync(join(dir, 'scripts'));
102 mkdirSync(join(dir, '.github/workflows'), { recursive: true });
103 copyFileSync(join(ROOT, 'stryker.conf.json'), join(dir, 'stryker.conf.json'));
104 writeFileSync(join(dir, 'src/a.ts'), lines([other]));
105 return dir;
106}
107const run = (cwd, ...args) => spawnSync(process.execPath, [SCRIPT, ...args], { cwd, env: { ...process.env, GITHUB_EVENT_NAME: 'push' }, encoding: 'utf8' });
109test('--weights stores seconds on the reference runner, scaled by each report\'s runner, and refuses an unscaled report', () => {
110 const dir = scratch();
111 try {
112 // 10 s on a 25 s runner is 20 s on the 50 s reference; 10 s on a 100 s runner is 5 s
113 writeFileSync(join(dir, 'fast.json'), shardReport(25_000, [mutant(2, 10_000)]));
114 writeFileSync(join(dir, 'slow.json'), shardReport(100_000, [mutant(1, 10_000)]));
115 const ok = run(dir, '--weights', 'fast.json', 'slow.json');
116 assert.equal(ok.status, 0, ok.stderr);
117 const w = JSON.parse(readFileSync(join(dir, 'scripts/mutate-weights.json'), 'utf8'))['src/a.ts'];
118 assert.deepEqual(w, { seconds: 25, mutants: 2, byLine: { 2: 20, 1: 5 } });
120 writeFileSync(join(dir, 'old.json'), shardReport(undefined, [mutant(2, 10_000)]));
121 const refused = run(dir, '--weights', 'old.json');
122 assert.equal(refused.status, 2, refused.stdout);
123 assert.match(refused.stderr, /without their runner's dry-run time/);
124 } finally {
125 rmSync(dir, { recursive: true, force: true });
126 }
127}, 60_000);
129test('--budget fails when the heaviest planned shard, on a runner twice as slow as the reference, passes 2/3 of mutate.yml\'s timeout-minutes', () => {
130 const dir = scratch();
131 try {
132 const yml = limit => `jobs:\n mutate-shard:\n timeout-minutes: ${limit}\n strategy:\n matrix:\n shard: [1, 2]\n mutate:\n timeout-minutes: 5\n`;
133 // src/a.ts at 3,000 s (all on line 2) on the reference runner is 2.15 + 3000 / 3.76 / 60 = 15.45 min in one shard
134 // (a unit is cut only at a line no mutant spans past, and the function's block mutant spans lines 1-3), 30.9 min on
135 // a 2x runner (RUNNER_WORST)
136 writeFileSync(join(dir, 'scripts/mutate-weights.json'), JSON.stringify({ 'src/a.ts': { seconds: 3000, mutants: 6, byLine: { 1: 0, 2: 3000, 3: 0 } } }));
137 writeFileSync(join(dir, '.github/workflows/mutate.yml'), yml(47)); // cap 2/3 of 47 = 31.3 min
138 const fits = run(dir, '--budget');
139 assert.equal(fits.status, 0, fits.stderr);
140 assert.match(fits.stdout, /in 2 shards; .* max 15\.4, .* on a 2x runner max 30\.9; cap 31\.3 \(67% of timeout-minutes 47\)/);
141 writeFileSync(join(dir, '.github/workflows/mutate.yml'), yml(46)); // cap 30.7 min
142 const over = run(dir, '--budget');
143 assert.equal(over.status, 1, over.stdout);
144 assert.match(over.stderr, /plans 15\.4 min, 30\.9 on a 2x runner, over 30\.7: src\/a\.ts/);
145 } finally {
146 rmSync(dir, { recursive: true, force: true });
147 }
148}, 60_000);
Diagnostics4 git calls · 522 ms
CallStatusSizeTime0522 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=4 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2003.2 KiB516 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=516 git.response.bytes=3302 git.response.cache=MISS git.response.duration_ms=516 git.fetch.wants=6a9f488aed19632a1ad2327da54de925ec869631 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3302 git.pkt.payload.max=3281 git.sideband.band1.bytes=3280 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2cff69b6fefad
- Request
- f345c49d…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format