GitHubivarvong

ply

main
ply6 git calls141 ms26 KiB3 cachedprotocol v2
148 lines8.4 KiBRaw
1// scripts/mutate.mjs on a pull request mutates every line the gate judges strictly, not only the lines the PR changed.
2// #198 moved parseHttpOptions and parseLsRefLine without changing their text: no line of theirs was in the diff, the PR's
3// mutation run never reached them, and main's full run found five survivors there (run 41976594897). Here a PR that
4// only moves a helper past a parser leaves the parser's lines out of the diff; the plan still holds all of them, and
5// still leaves out an ungated function the PR did not change.
6import { test } from 'vitest';
7import assert from 'node:assert/strict';
8import { spawnSync } from 'node:child_process';
9import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
10import { tmpdir } from 'node:os';
11import { join } from 'node:path';
12import { fileURLToPath } from 'node:url';
14const ROOT = fileURLToPath(new URL('..', import.meta.url));
15const SCRIPT = join(ROOT, 'scripts/mutate.mjs');
17// parseFrame is gated (PARSER); helper and other are not. 40 statements, so git's diff moves the 3-line helper, not
18// the parser.
19const parser = [
20 'export function parseFrame(b: Uint8Array): number {',
21 ' let n = 0;',
22 ...Array.from({ length: 40 }, (_, i) => ` if (b[${i}] === ${i + 1}) n += ${i + 2};`),
23 ' return n;',
24 '}',
25];
26const helper = ['export function helper(x: number): number {', ' return x > 1 ? x * 2 : x - 3;', '}'];
27const other = ['export function other(x: number): number {', ' return x < 4 ? x + 5 : x * 6;', '}'];
28const lines = parts => parts.flat().join('\n') + '\n';
30function git(cwd, ...args) {
31 const r = spawnSync('git', args, { cwd, encoding: 'utf8' });
32 assert.equal(r.status, 0, `git ${args.join(' ')}: ${r.stderr}`);
33 return r.stdout.trim();
36function prPlan(cwd, base) {
37 const env = { ...process.env, GITHUB_EVENT_NAME: 'pull_request', MUTATE_BASE: base };
38 delete env.MUTATE_FULL;
39 const r = spawnSync(process.execPath, [SCRIPT, '--plan', '4'], { cwd, env, encoding: 'utf8' });
40 assert.equal(r.status, 0, r.stderr);
41 return JSON.parse(r.stdout);
44// [from, to] ranges of a file's units in a plan
45const unitRanges = (plan, file) => plan.shards.flatMap(s => s.files)
46 .filter(u => u === file || u.startsWith(`${file}:`))
47 .map(u => (u === file ? [1, Infinity] : u.slice(file.length + 1).split('-').map(Number)));
48const covers = (ranges, from, to) => ranges.some(([a, b]) => a <= from && b >= to);
50test('a pull request that only moves code past a parser still mutates every line of the parser', () => {
51 const dir = mkdtempSync(join(tmpdir(), 'ply-mutate-plan-'));
52 try {
53 mkdirSync(join(dir, 'src'));
54 copyFileSync(join(ROOT, 'stryker.conf.json'), join(dir, 'stryker.conf.json'));
55 git(dir, 'init', '-q', '-b', 'main');
56 git(dir, 'config', 'user.email', 't@example.com');
57 git(dir, 'config', 'user.name', 't');
58 writeFileSync(join(dir, 'src/a.ts'), lines([helper, parser, other]));
59 git(dir, 'add', '.');
60 git(dir, 'commit', '-q', '-m', 'base');
61 const base = git(dir, 'rev-parse', 'HEAD');
62 writeFileSync(join(dir, 'src/a.ts'), lines([parser, helper, other]));
63 git(dir, 'commit', '-q', '-am', 'move helper below the parser');
65 // the parser's lines on the PR side: 1..parser.length. The changed lines (a pure deletion borders the lines on
66 // either side of it, as in changedRanges) do not cover them: changed lines alone would not mutate the parser.
67 const from = 1, to = parser.length;
68 const hunks = git(dir, 'diff', '-U0', `${base}...HEAD`).split('\n').filter(l => l.startsWith('@@'));
69 const changed = hunks.map(h => /\+(\d+)(?:,(\d+))?/.exec(h)).map(m => {
70 const at = Number(m[1]), len = Number(m[2] ?? 1);
71 return len === 0 ? [Math.max(1, at), at + 1] : [at, at + len - 1];
72 });
73 assert.ok(!covers(changed, from, to), `the diff changes the parser: ${hunks.join(' ')}`);
74 assert.ok(changed.every(([a, b]) => b <= from || a > to), `the diff changes the parser: ${hunks.join(' ')}`);
76 const plan = prPlan(dir, base);
77 assert.equal(plan.full, false);
78 const ranges = unitRanges(plan, 'src/a.ts');
79 assert.ok(covers(ranges, from, to), `parseFrame ${from}-${to} not in the plan: ${JSON.stringify(ranges)}`);
80 // the moved helper is there too (changed lines), the untouched ungated function is not
81 const helperAt = parser.length + 1;
82 assert.ok(covers(ranges, helperAt, helperAt + helper.length - 1), JSON.stringify(ranges));
83 const otherAt = parser.length + helper.length + 1;
84 assert.ok(!ranges.some(([a, b]) => a <= otherAt + 1 && b >= otherAt + 1), `other() is in the plan: ${JSON.stringify(ranges)}`);
85 // each planned mutant once: the parser's 40 conditions alone are well over 40
86 assert.ok(plan.mutants > 40, String(plan.mutants));
87 } finally {
88 rmSync(dir, { recursive: true, force: true });
89 }
90}, 60_000);
92// #313: a shard of 43232620788 planned at 9,322 s ran 19,065 s (84 of 90 minutes), because its weights were measured on
93// a runner whose dry run took 28 s and it ran on one taking 50 s. --weights scales every duration to the reference
94// runner by the dry-run time of the runner that measured it, and refuses a report that does not say.
95const mutant = (line, ms) => ({ id: String(line), mutatorName: 'BooleanLiteral', replacement: 'true', status: 'Killed', duration: ms, location: { start: { line, column: 3 }, end: { line, column: 9 } } });
96const shardReport = (dryRunNetMs, mutants) => JSON.stringify({ shard: { i: 1, n: 1, files: ['src/a.ts'], dryRunNetMs }, files: { 'src/a.ts': { mutants } } });
98function scratch() {
99 const dir = mkdtempSync(join(tmpdir(), 'ply-mutate-weights-'));
100 mkdirSync(join(dir, 'src'));
101 mkdirSync(join(dir, 'scripts'));
102 mkdirSync(join(dir, '.github/workflows'), { recursive: true });
103 copyFileSync(join(ROOT, 'stryker.conf.json'), join(dir, 'stryker.conf.json'));
104 writeFileSync(join(dir, 'src/a.ts'), lines([other]));
105 return dir;
107const run = (cwd, ...args) => spawnSync(process.execPath, [SCRIPT, ...args], { cwd, env: { ...process.env, GITHUB_EVENT_NAME: 'push' }, encoding: 'utf8' });
109test('--weights stores seconds on the reference runner, scaled by each report\'s runner, and refuses an unscaled report', () => {
110 const dir = scratch();
111 try {
112 // 10 s on a 25 s runner is 20 s on the 50 s reference; 10 s on a 100 s runner is 5 s
113 writeFileSync(join(dir, 'fast.json'), shardReport(25_000, [mutant(2, 10_000)]));
114 writeFileSync(join(dir, 'slow.json'), shardReport(100_000, [mutant(1, 10_000)]));
115 const ok = run(dir, '--weights', 'fast.json', 'slow.json');
116 assert.equal(ok.status, 0, ok.stderr);
117 const w = JSON.parse(readFileSync(join(dir, 'scripts/mutate-weights.json'), 'utf8'))['src/a.ts'];
118 assert.deepEqual(w, { seconds: 25, mutants: 2, byLine: { 2: 20, 1: 5 } });
120 writeFileSync(join(dir, 'old.json'), shardReport(undefined, [mutant(2, 10_000)]));
121 const refused = run(dir, '--weights', 'old.json');
122 assert.equal(refused.status, 2, refused.stdout);
123 assert.match(refused.stderr, /without their runner's dry-run time/);
124 } finally {
125 rmSync(dir, { recursive: true, force: true });
126 }
127}, 60_000);
129test('--budget fails when the heaviest planned shard, on a runner twice as slow as the reference, passes 2/3 of mutate.yml\'s timeout-minutes', () => {
130 const dir = scratch();
131 try {
132 const yml = limit => `jobs:\n mutate-shard:\n timeout-minutes: ${limit}\n strategy:\n matrix:\n shard: [1, 2]\n mutate:\n timeout-minutes: 5\n`;
133 // src/a.ts at 3,000 s (all on line 2) on the reference runner is 2.15 + 3000 / 3.76 / 60 = 15.45 min in one shard
134 // (a unit is cut only at a line no mutant spans past, and the function's block mutant spans lines 1-3), 30.9 min on
135 // a 2x runner (RUNNER_WORST)
136 writeFileSync(join(dir, 'scripts/mutate-weights.json'), JSON.stringify({ 'src/a.ts': { seconds: 3000, mutants: 6, byLine: { 1: 0, 2: 3000, 3: 0 } } }));
137 writeFileSync(join(dir, '.github/workflows/mutate.yml'), yml(47)); // cap 2/3 of 47 = 31.3 min
138 const fits = run(dir, '--budget');
139 assert.equal(fits.status, 0, fits.stderr);
140 assert.match(fits.stdout, /in 2 shards; .* max 15\.4, .* on a 2x runner max 30\.9; cap 31\.3 \(67% of timeout-minutes 47\)/);
141 writeFileSync(join(dir, '.github/workflows/mutate.yml'), yml(46)); // cap 30.7 min
142 const over = run(dir, '--budget');
143 assert.equal(over.status, 1, over.stdout);
144 assert.match(over.stderr, /plans 15\.4 min, 30\.9 on a 2x runner, over 30\.7: src\/a\.ts/);
145 } finally {
146 rmSync(dir, { recursive: true, force: true });
147 }
148}, 60_000);
Diagnostics6 git calls · 141 ms
CallStatusSizeTime0141 ms
discover200191 B4 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=4
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB4 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=4
git.response.bytes=11990
git.response.cache=UPDATING
git.response.duration_ms=4
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2003.2 KiB129 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=129
git.response.bytes=3299
git.response.cache=MISS
git.response.duration_ms=129
git.fetch.wants=6a9f488aed19632a1ad2327da54de925ec869631
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3299
git.pkt.payload.max=3272
git.sideband.band1.bytes=3272
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2ad213f91e6b0
Request
6ecf2201…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format