// A workbench run as a deterministic test for ply's own probe suite: the scenario JSON (the plan verbatim, the run's // observation, and the differences the workbench derived from it) and a vitest file that sits beside it in a ply // checkout (test/probes/replay/). The file runs the plan the way test/probes/probes.test.js does, with ply and stock // git against git http-backend under Artifacts' policy (the reference), and asserts: // 1. the plan is ply's catalogue entry at that commit (a changed plan is a different probe), // 2. both reference runs meet the stated expectation and agree (a failure is a client or harness problem), // 3. the recorded observation, evaluated by that commit's ply/probes soloDiffs, yields the differences recorded here. // It holds what the server answered and nothing else: no classification, no credential, no repository URL. import type { Difference, Names, Observation, Scenario } from 'ply/probes'; import type { RunId, RunSeed } from './brand.ts'; export const REPLAY_FORMAT = 'ply-probe-replay/1'; export type Replay = { readonly format: typeof REPLAY_FORMAT; readonly run: RunId; readonly seed: RunSeed; readonly ply: string; // the ply commit the Worker ran readonly scenario: Scenario; readonly observed: Observation; readonly names: Names; // oid -> seed name: the same ids every run of the plan builds readonly differences: ReadonlyArray>; readonly cfRays: readonly string[]; // every Artifacts exchange of the run, in order }; export const replaySlug = (r: Pick): string => `${r.scenario.id.replace(/[^a-z0-9]+/g, '-')}-${r.run.slice(0, 8)}`; export function replayJson(r: Replay): string { return `${JSON.stringify(r, null, 2)}\n`; } export function replayTest(r: Replay): string { const slug = replaySlug(r); return `// Replay of a ply workbench run: scenario ${r.scenario.id}, run ${r.run}, seed ${r.seed}, ply ${r.ply}. // Generated by ply-web /bench (src/bench/export.ts). Put this file and ${slug}.json in test/probes/replay/ of a ply // checkout and run \`npx vitest run test/probes/replay/${slug}.test.js\`. The reference is git http-backend with // Artifacts' policy on 127.0.0.1 (test/probes/runner.mjs); nothing here reaches the network. import { test, beforeAll, afterAll } from 'vitest'; import assert from 'node:assert/strict'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { SCENARIOS } from '../../../src/probe-scenarios.ts'; import { ALLOW } from '../../../src/probe-allow.ts'; import { compareScenario, evaluablePairs, judge, runsFor, soloDiffs } from '../../../src/probe-evaluate.ts'; import { startReference, runScenario, newTracer } from '../runner.mjs'; const replay = JSON.parse(readFileSync(new URL('./${slug}.json', import.meta.url), 'utf8')); const s = replay.scenario; const plain = v => JSON.parse(JSON.stringify(v)); test('${r.scenario.id}: the plan is the catalogue entry at this commit', () => { assert.equal(replay.format, '${REPLAY_FORMAT}'); const now = SCENARIOS.find(x => x.id === s.id); assert.ok(now, \`\${s.id} is not in the catalogue at this commit\`); assert.deepEqual(plain(now), s, 'the plan changed since the workbench ran it: this replay is a different probe'); }); let ref; beforeAll(async () => { ref = await startReference(); }); afterAll(async () => { await ref.close(); }); test('${r.scenario.id}: ply and stock git against the reference meet the expectation and agree', async () => { const runs = {}; for (const t of ['ref-ply', 'ref-git'].filter(x => runsFor(s, x === 'ref-git' ? 'git' : 'ply'))) { const root = mkdtempSync(join(tmpdir(), 'ply-replay-')); try { const repo = await ref.repo(s); const r = await runScenario(s, { name: t, client: t.slice(4), proxy: ref.proxy, repoPath: repo.repoPath }, { tracer: newTracer().tracer, root }); runs[t] = r.observation; assert.deepEqual(r.byOid, replay.names, 'the seed builds the same object ids as the workbench run'); for (const st of r.observation.steps) for (const o of st.outcomes) assert.doesNotMatch(o.detail, /never fired/, \`\${s.id} \${t}\`); } finally { rmSync(root, { recursive: true, force: true }); } } const v = judge(SCENARIOS, compareScenario(s, runs, replay.names), ALLOW, [{ scenario: s.id, pairs: evaluablePairs(runs) }]); assert.deepEqual(v.findings, [], 'the reference runs disagree with the expectation or with each other'); assert.deepEqual(soloDiffs(s, runs['ref-ply'], replay.names), [], 'ply against the reference misses the stated expectation'); }, 120_000); test('${r.scenario.id}: the recorded server run yields the recorded differences', () => { const got = soloDiffs(s, replay.observed, replay.names).map(d => ({ pair: d.pair, field: d.field, left: d.left, right: d.right })); assert.deepEqual(got, replay.differences); }); `; }