GitHubivarvong
ply
main
1// Differential probes (t229u days 3-4): every scenario in src/probe-scenarios.ts against the reference (git
2// http-backend with Artifacts' known policy) by ply and by stock git, and with --artifacts against Cloudflare Artifacts
3// too; compared by src/probe-evaluate.ts; expected differences from src/probe-allow.ts.
4//
5// node --import ./test/ts-resolve.mjs scripts/probes.mjs [--only <substring>[,...]] reference only
6// PLY_PROBES_CF=<cf binary> PLY_PROBES_CF_CWD=<dir> \
7// node --import ./test/ts-resolve.mjs scripts/probes.mjs --artifacts --out <private dir> [--only <substring>]
8//
9// --artifacts is opt-in and private: one fresh repository per scenario run, in the ply-loadgen namespace only, one at a
10// time; results (outcomes, findings, cf-ray of every request) go to <private dir>/<run-id>.json, which must be outside
11// this repository. CI runs the reference only (test/probes/probes.test.js). Exit 1 on any finding, unused allow entry,
12// run error, token not proven revoked, or repository not deleted.
13import { mkdtempSync, rmSync, writeFileSync, mkdirSync, readFileSync } from 'node:fs';
14import { createHash } from 'node:crypto';
15import { tmpdir } from 'node:os';
16import { join, resolve, relative } from 'node:path';
17import { execFileSync } from 'node:child_process';
18import { fileURLToPath } from 'node:url';
19import { SCENARIOS } from '../src/probe-scenarios.ts';
20import { ALLOW } from '../src/probe-allow.ts';
21import { compareScenario, evaluablePairs, judge, runsFor } from '../src/probe-evaluate.ts';
22import { startReference, runScenario, newTracer } from '../test/probes/runner.mjs';
24const ROOT = fileURLToPath(new URL('..', import.meta.url));
25const arg = name => { const i = process.argv.indexOf(name); return i < 0 ? null : process.argv[i + 1] ?? null; };
26const artifacts = process.argv.includes('--artifacts');
27const only = arg('--only');
28const outDir = arg('--out');
29if (artifacts && (outDir === null || !relative(ROOT, resolve(outDir)).startsWith('..'))) { console.error('--artifacts needs --out <dir> outside this repository (results are private)'); process.exit(2); }
30const stamp = new Date().toISOString().replace(/[-:]/g, '').toLowerCase().slice(0, 13); // UTC, e.g. 20261114t1737
31const runId = arg('--run-id') ?? `${stamp}-${Math.random().toString(36).slice(2, 6)}`;
32const plyCommit = (() => { try { return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: ROOT, encoding: 'utf8' }).trim(); } catch { return null; } })();
33const gitVersion = execFileSync('git', ['--version'], { encoding: 'utf8' }).trim();
34// The probe code that ran, as one hash (the run may be made before the commit that cites it).
35const SOURCES = ['src/probe-types.ts', 'src/probe-scenarios.ts', 'src/probe-evaluate.ts', 'src/probe-allow.ts', 'test/probes/git-outcome.ts', 'test/probes/runner.mjs', 'test/probes/artifacts.mjs', 'scripts/probes.mjs'];
36const sourceHash = createHash('sha256').update(SOURCES.map(f => readFileSync(join(ROOT, f))).reduce((a, b) => Buffer.concat([a, b]), Buffer.alloc(0))).digest('hex').slice(0, 16);
38const audit = [];
39let art = null, scrub = s => String(s);
40if (artifacts) {
41 const bin = process.env.PLY_PROBES_CF, cwd = process.env.PLY_PROBES_CF_CWD;
42 if (!bin || !cwd) { console.error('--artifacts needs PLY_PROBES_CF (the cf CLI) and PLY_PROBES_CF_CWD'); process.exit(2); }
43 const env = { ...process.env }; delete env.CLOUDFLARE_API_TOKEN;
44 const mod = await import('../test/probes/artifacts.mjs');
45 scrub = mod.scrub;
46 art = mod.artifactsTarget({ cli: { bin, cwd, env }, runId, log: r => audit.push(r) });
47}
49const scenarios = SCENARIOS.filter(s => only === null || only.split(',').some(o => s.id.includes(o)));
50const targets = artifacts ? ['ref-ply', 'ref-git', 'art-ply', 'art-git'] : ['ref-ply', 'ref-git'];
51const ref = await startReference();
52const results = [], diffs = [], evaluated = [], errors = [];
53// An interrupted run still revokes the open repository's tokens and deletes it. `open` is set before the repository
54// is created (a signal during art.open used to leave it behind, run 20261114t2200-full) and `closing` until the close
55// has finished, so a signal at any point waits for the repository to be opened and then closes it.
56let open = null, closing = null;
57for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.once(sig, async () => {
58 console.error(`${sig}: closing the open repository`);
59 if (open !== null) { const h = await open.catch(() => null); if (h !== null) await h.close(); }
60 if (closing !== null) await closing.catch(() => {});
61 process.exit(130);
62});
63const t0 = Date.now();
64for (const s of scenarios) {
65 const runs = {}, record = { id: s.id, targets: {} };
66 let names = {};
67 // a scenario one client cannot send at all is not run for it (no repository is created for it)
68 for (const t of targets.filter(x => runsFor(s, x.endsWith('-git') ? 'git' : 'ply'))) {
69 const root = mkdtempSync(join(tmpdir(), 'ply-probe-'));
70 const { tracer, spans } = newTracer();
71 let handle = null, proxy = ref.proxy, repoPath, before = ref.proxy.exchanges.length;
72 const started = Date.now();
73 try {
74 if (t.startsWith('art-')) { open = art.open(`${s.id}/${t}`); handle = await open; proxy = handle.proxy; repoPath = handle.repoPath; before = 0; }
75 else repoPath = (await ref.repo(s)).repoPath;
76 const r = await runScenario(s, { name: t, client: t.slice(4), proxy, repoPath }, { tracer, root });
77 runs[t] = r.observation; names = r.byOid;
78 record.targets[t] = { ms: Date.now() - started, repo: handle?.name ?? repoPath, observation: r.observation, stepExchanges: r.stepExchanges };
79 } catch (e) {
80 errors.push({ scenario: s.id, target: t, error: scrub(e?.message ?? e) });
81 record.targets[t] = { ms: Date.now() - started, repo: handle?.name ?? repoPath, error: scrub(e?.message ?? e) };
82 } finally {
83 record.targets[t].exchanges = proxy.exchanges.slice(before);
84 record.targets[t].spans = spans();
85 open = null;
86 if (handle !== null) { closing = handle.close(); await closing; closing = null; }
87 rmSync(root, { recursive: true, force: true });
88 }
89 const st = record.targets[t];
90 const shown = st.error ? `ERROR ${st.error}` : st.observation.steps.map(x => x.outcomes.map(o => o.outcome).join(',')).join(' ; ');
91 console.log(`${s.id.padEnd(58)} ${t.padEnd(8)} ${String(st.ms).padStart(6)} ms ${shown}`);
92 }
93 record.names = names;
94 const d = compareScenario(s, runs, names);
95 diffs.push(...d);
96 evaluated.push({ scenario: s.id, pairs: evaluablePairs(runs) });
97 results.push(record);
98}
99await ref.close();
100const verdict = judge(SCENARIOS, diffs, ALLOW, evaluated);
101const tokensOk = art === null || art.tokens.every(t => t.revoked && (t.label === 'leftover' || t.probe === 403));
102const reposOk = art === null || art.repos.every(r => r.deleted && r.listedAfter === false);
104console.log(`\n${scenarios.length} scenarios x ${targets.length} targets in ${((Date.now() - t0) / 1000).toFixed(0)} s; ${diffs.length} differences: ${verdict.expected.length} expected, ${verdict.findings.length} findings; ${verdict.unused.length} unused allow entries; ${errors.length} run errors`);
105for (const f of verdict.findings) console.log(` FINDING [${f.class}] ${f.scenario} ${f.pair} ${f.field}: ${f.left.slice(0, 160)} vs ${f.right.slice(0, 160)} (${f.why})`);
106for (const e of verdict.expected) console.log(` expected ${e.difference.scenario} ${e.difference.pair} ${e.difference.field}: ${e.reason}`);
107for (const a of verdict.unused) console.log(` UNUSED allow entry ${a.scenario} ${a.pair} ${a.field}`);
108for (const e of errors) console.log(` ERROR ${e.scenario} ${e.target}: ${e.error}`);
109if (art !== null) console.log(`tokens: ${art.tokens.length} minted (auto ${art.tokens.filter(t => t.label.startsWith('auto')).length}), revoked ${art.tokens.filter(t => t.revoked).length}, proven 403 ${art.tokens.filter(t => t.probe === 403).length}; repos: ${art.repos.length} created, ${art.repos.filter(r => r.deleted).length} deleted, ${art.repos.filter(r => r.listedAfter === false).length} confirmed gone`);
111if (outDir !== null) {
112 mkdirSync(outDir, { recursive: true });
113 const checks = artifacts ? scenarios.flatMap(s => ['art-ply', 'art-git'].filter(t => runsFor(s, t === 'art-git' ? 'git' : 'ply')).map(t => `${s.id}/${t}`)) : [];
114 // each finding with the cf-ray of every request the client under test made in the scenario's steps, per target
115 const raysOf = (id, t) => (results.find(x => x.id === id)?.targets[t]?.stepExchanges ?? []).flat().filter(e => e.channel === 'main').map(e => ({ method: e.method, status: e.status, cfRay: e.cfRay, fault: e.fault }));
116 const findings = verdict.findings.map(f => ({ ...f, rays: Object.fromEntries(f.pair.split('~').filter(t => t.startsWith('art-')).map(t => [t, raysOf(f.scenario, t)])) }));
117 const out = { runId: `probes-${runId}`, startedAt: new Date(t0).toISOString(), plyCommit, sourceHash, gitVersion, node: process.version, targets, checks, verdict: { ...verdict, findings }, differences: diffs, errors, tokens: art?.tokens ?? [], repos: art?.repos ?? [], controlPlane: audit, scenarios: results };
118 const file = join(outDir, `probes-${runId}.json`);
119 writeFileSync(file, scrub(JSON.stringify(out, null, 1)), { mode: 0o600 });
120 console.log(`results: ${file}`);
121}
122process.exit(verdict.findings.length || verdict.unused.length || errors.length || !tokensOk || !reposOk ? 1 : 0);
Diagnostics6 git calls · 99 ms
CallStatusSizeTime099 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=11990 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:0200831 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=90 git.response.bytes=831 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=b9290e370bd091fd3272931655a175dd1a7af82d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=831 git.pkt.payload.max=804 git.sideband.band1.bytes=804 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=804 git.pack.trailer=81ddc1f73214935b4ee953ad84f959db955ce0d1Open pack map
fetch2003.7 KiB90 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3773 git.response.cache=MISS git.response.duration_ms=90 git.fetch.wants=b6be7bb3f939dec3b5ce15b01811b2b3917d1335 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3773 git.pkt.payload.max=3746 git.sideband.band1.bytes=3746 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e15f5e5e23725b
- Request
- c2d09c46…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format