main
1// A workbench run as a deterministic test for ply's own probe suite: the scenario JSON (the plan verbatim, the run's
2// observation, and the differences the workbench derived from it) and a vitest file that sits beside it in a ply
3// checkout (test/probes/replay/). The file runs the plan the way test/probes/probes.test.js does, with ply and stock
4// git against git http-backend under Artifacts' policy (the reference), and asserts:
5// 1. the plan is ply's catalogue entry at that commit (a changed plan is a different probe),
6// 2. both reference runs meet the stated expectation and agree (a failure is a client or harness problem),
7// 3. the recorded observation, evaluated by that commit's ply/probes soloDiffs, yields the differences recorded here.
8// It holds what the server answered and nothing else: no classification, no credential, no repository URL.
9import type { Difference, Names, Observation, Scenario } from 'ply/probes';
10import type { RunId, RunSeed } from './brand.ts';
12export const REPLAY_FORMAT = 'ply-probe-replay/1';
13export type Replay = {
14 readonly format: typeof REPLAY_FORMAT;
15 readonly run: RunId;
16 readonly seed: RunSeed;
17 readonly ply: string; // the ply commit the Worker ran
18 readonly scenario: Scenario;
19 readonly observed: Observation;
20 readonly names: Names; // oid -> seed name: the same ids every run of the plan builds
21 readonly differences: ReadonlyArray<Pick<Difference, 'pair' | 'field' | 'left' | 'right'>>;
22 readonly cfRays: readonly string[]; // every Artifacts exchange of the run, in order
23};
25export const replaySlug = (r: Pick<Replay, 'scenario' | 'run'>): string => `${r.scenario.id.replace(/[^a-z0-9]+/g, '-')}-${r.run.slice(0, 8)}`;
27export function replayJson(r: Replay): string {
28 return `${JSON.stringify(r, null, 2)}\n`;
29}
31export function replayTest(r: Replay): string {
32 const slug = replaySlug(r);
33 return `// Replay of a ply workbench run: scenario ${r.scenario.id}, run ${r.run}, seed ${r.seed}, ply ${r.ply}.
34// Generated by ply-web /bench (src/bench/export.ts). Put this file and ${slug}.json in test/probes/replay/ of a ply
35// checkout and run \`npx vitest run test/probes/replay/${slug}.test.js\`. The reference is git http-backend with
36// Artifacts' policy on 127.0.0.1 (test/probes/runner.mjs); nothing here reaches the network.
37import { test, beforeAll, afterAll } from 'vitest';
38import assert from 'node:assert/strict';
39import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
40import { tmpdir } from 'node:os';
41import { join } from 'node:path';
42import { SCENARIOS } from '../../../src/probe-scenarios.ts';
43import { ALLOW } from '../../../src/probe-allow.ts';
44import { compareScenario, evaluablePairs, judge, runsFor, soloDiffs } from '../../../src/probe-evaluate.ts';
45import { startReference, runScenario, newTracer } from '../runner.mjs';
47const replay = JSON.parse(readFileSync(new URL('./${slug}.json', import.meta.url), 'utf8'));
48const s = replay.scenario;
49const plain = v => JSON.parse(JSON.stringify(v));
51test('${r.scenario.id}: the plan is the catalogue entry at this commit', () => {
52 assert.equal(replay.format, '${REPLAY_FORMAT}');
53 const now = SCENARIOS.find(x => x.id === s.id);
54 assert.ok(now, \`\${s.id} is not in the catalogue at this commit\`);
55 assert.deepEqual(plain(now), s, 'the plan changed since the workbench ran it: this replay is a different probe');
56});
58let ref;
59beforeAll(async () => { ref = await startReference(); });
60afterAll(async () => { await ref.close(); });
62test('${r.scenario.id}: ply and stock git against the reference meet the expectation and agree', async () => {
63 const runs = {};
64 for (const t of ['ref-ply', 'ref-git'].filter(x => runsFor(s, x === 'ref-git' ? 'git' : 'ply'))) {
65 const root = mkdtempSync(join(tmpdir(), 'ply-replay-'));
66 try {
67 const repo = await ref.repo(s);
68 const r = await runScenario(s, { name: t, client: t.slice(4), proxy: ref.proxy, repoPath: repo.repoPath }, { tracer: newTracer().tracer, root });
69 runs[t] = r.observation;
70 assert.deepEqual(r.byOid, replay.names, 'the seed builds the same object ids as the workbench run');
71 for (const st of r.observation.steps) for (const o of st.outcomes) assert.doesNotMatch(o.detail, /never fired/, \`\${s.id} \${t}\`);
72 } finally { rmSync(root, { recursive: true, force: true }); }
73 }
74 const v = judge(SCENARIOS, compareScenario(s, runs, replay.names), ALLOW, [{ scenario: s.id, pairs: evaluablePairs(runs) }]);
75 assert.deepEqual(v.findings, [], 'the reference runs disagree with the expectation or with each other');
76 assert.deepEqual(soloDiffs(s, runs['ref-ply'], replay.names), [], 'ply against the reference misses the stated expectation');
77}, 120_000);
79test('${r.scenario.id}: the recorded server run yields the recorded differences', () => {
80 const got = soloDiffs(s, replay.observed, replay.names).map(d => ({ pair: d.pair, field: d.field, left: d.left, right: d.right }));
81 assert.deepEqual(got, replay.differences);
82});
83`;
84}
Diagnostics4 git calls · 233 ms
CallStatusSizeTime0233 ms
discover200152 B13 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=4 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=217 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2002.1 KiB217 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=2112 git.response.cache=MISS git.response.duration_ms=217 git.fetch.wants=3d1aa81ac303905e06379f2a4906e41e69884905 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2112 git.pkt.payload.max=2091 git.sideband.band1.bytes=2090 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2d9d10f3f14c2
- Request
- b82e693d…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format