main
1// The Git behaviour coverage matrix, from the command line. Reads test/behaviours/matrix.ts (the behaviours) and
2// test/behaviours/runs.ts (the recorded runs they cite).
3//
4// npm run coverage-matrix counts by area, status and evidence kind, then the gaps
5// npm run coverage-matrix -- --md regenerate COVERAGE.md (test/behaviours/matrix.test.js fails if it drifts)
6import { writeFileSync } from 'node:fs';
7import { join } from 'node:path';
8import { fileURLToPath } from 'node:url';
9import { BEHAVIOURS } from '../test/behaviours/matrix.ts';
10import { RUNS } from '../test/behaviours/runs.ts';
11import { AREAS, AREA_TITLES, EVIDENCE_KINDS } from '../test/behaviours/types.ts';
13const ROOT = join(fileURLToPath(import.meta.url), '..', '..');
14const STATUSES = ['verified', 'implemented', 'unsupported'];
15const ARTIFACTS_KINDS = ['ply_vs_artifacts', 'native_git_vs_artifacts', 'wire_probe_vs_artifacts'];
16const SHORT = { ply_vs_reference_git: 'ply↔git', ply_vs_artifacts: 'ply↔Artifacts', native_git_vs_artifacts: 'git↔Artifacts', wire_probe_vs_artifacts: 'wire↔Artifacts' };
18const esc = s => String(s).replace(/\|/g, '\\|').replace(/\n/g, ' ').replace(/</g, '<');
19// Every piece of evidence a behaviour carries: a verified status's evidence, or what others observed of an unsupported one.
20export const evidenceOf = b => b.status.tag === 'verified' ? b.status.evidence : b.status.tag === 'unsupported' ? b.status.observed : [];
21const kindsOf = b => new Set(evidenceOf(b).map(e => e.kind));
22const where = e => e.ref.tag === 'test' ? `\`${e.ref.path}\`` : `${e.ref.run}: \`${esc(e.ref.check)}\``;
24// The gap classes, worst first. A behaviour is in the first class that holds for it.
25export const GAPS = [
26 { key: 'implemented', title: 'Exercised by ply, judged by nothing independent', holds: b => b.status.tag === 'implemented' },
27 { key: 'no-artifacts', title: 'Verified against git http-backend only: no evidence of any kind from Artifacts', holds: b => b.status.tag === 'verified' && !ARTIFACTS_KINDS.some(k => kindsOf(b).has(k)) },
28 { key: 'no-ply-on-artifacts', title: 'Artifacts observed by stock git or wire probes, never by ply', holds: b => b.status.tag === 'verified' && !kindsOf(b).has('ply_vs_artifacts') },
29 { key: 'no-reference', title: 'Verified on Artifacts, but ply\'s probe is not checked against git http-backend', holds: b => b.status.tag === 'verified' && !kindsOf(b).has('ply_vs_reference_git') },
30 { key: 'unsupported', title: 'Not exercisable by ply', holds: b => b.status.tag === 'unsupported' },
31];
32export const gapOf = b => GAPS.find(g => g.holds(b));
34export function counts(all = BEHAVIOURS) {
35 const byArea = AREAS.map(area => {
36 const bs = all.filter(b => b.area === area);
37 return { area, total: bs.length, ...Object.fromEntries(STATUSES.map(s => [s, bs.filter(b => b.status.tag === s).length])) };
38 });
39 const verified = all.filter(b => b.status.tag === 'verified');
40 const byKind = Object.fromEntries(EVIDENCE_KINDS.map(k => [k, verified.filter(b => kindsOf(b).has(k)).length]));
41 const anyArtifacts = verified.filter(b => ARTIFACTS_KINDS.some(k => kindsOf(b).has(k))).length;
42 const gaps = Object.fromEntries(GAPS.map(g => [g.key, all.filter(b => gapOf(b) === g).length]));
43 return { total: all.length, byArea, byKind, anyArtifacts, gaps, ...Object.fromEntries(STATUSES.map(s => [s, all.filter(b => b.status.tag === s).length])) };
44}
46export function renderMd(all = BEHAVIOURS, runs = RUNS) {
47 const c = counts(all);
48 const L = [];
49 L.push('# Git behaviour coverage', '');
50 L.push('<!-- generated by scripts/coverage-matrix.mjs --md from test/behaviours/matrix.ts and test/behaviours/runs.ts; test/behaviours/matrix.test.js fails if it drifts -->', '');
51 L.push('ply is a Git protocol conformance and behaviour-testing instrument for Cloudflare Artifacts as well as a client. This matrix counts **server behaviours**, not git commands: each row is a behaviour or failure mode of a Git server, the ply operations and tests that exercise it, and who has independently checked the server\'s answer.', '');
52 L.push('- **unsupported**: ply has no operation that exercises it. Stock git or hand-built wire probes may still have observed Artifacts.');
53 L.push('- **implemented**: ply exercises it, but only against fakes or its own expectations; nothing independent has judged the server.');
54 L.push('- **verified**: at least one independent check. The types make a verified row without evidence impossible.', '');
55 L.push('Evidence kinds:', '');
56 L.push('- `ply_vs_reference_git` (ply↔git): ply against git http-backend in this suite, with git\'s own output as the expected value; the cited file runs real git.');
57 L.push('- `ply_vs_artifacts` (ply↔Artifacts): ply against Artifacts in a recorded run.');
58 L.push('- `native_git_vs_artifacts` (git↔Artifacts): stock git against Artifacts in a recorded run, with git http-backend beside it as the baseline.');
59 L.push('- `wire_probe_vs_artifacts` (wire↔Artifacts): hand-built requests (pkt-lines, packs, HTTP framing) against Artifacts beside the same baseline. Neither ply nor stock git sends these bytes.', '');
60 L.push('Only Artifacts evidence says anything about Artifacts; reference evidence says ply\'s probe is right. Recorded runs are private (their harnesses hold credentials): `test/behaviours/runs.ts` keeps each run\'s id, date and check names, never its verdicts.', '');
61 L.push('## Summary', '');
62 L.push(`**${c.total}** behaviours: **${c.verified}** verified, **${c.implemented}** implemented, **${c.unsupported}** unsupported. Of the verified, **${c.anyArtifacts}** carry some Artifacts evidence and **${c.byKind.ply_vs_artifacts}** carry evidence of ply itself against Artifacts.`, '');
63 L.push('| area | behaviours | verified | implemented | unsupported |', '|---|---:|---:|---:|---:|');
64 for (const a of c.byArea) L.push(`| ${a.area} | ${a.total} | ${a.verified} | ${a.implemented} | ${a.unsupported} |`);
65 L.push(`| **all** | **${c.total}** | **${c.verified}** | **${c.implemented}** | **${c.unsupported}** |`, '');
66 L.push('Verified behaviours by evidence kind (a behaviour counts once under each kind it has):', '');
67 L.push('| evidence kind | verified behaviours |', '|---|---:|');
68 for (const k of EVIDENCE_KINDS) L.push(`| \`${k}\` | ${c.byKind[k]} |`);
69 L.push('');
70 L.push('## Gaps', '');
71 L.push('Each behaviour that is not verified by both ply↔git and ply↔Artifacts, filed under the first of these classes that holds:', '');
72 for (const g of GAPS) {
73 const bs = all.filter(b => gapOf(b) === g);
74 L.push(`### ${g.title} (${bs.length})`, '');
75 if (bs.length === 0) { L.push('None.', ''); continue; }
76 for (const b of bs) L.push(`- \`${b.id}\`: ${esc(b.description)}`);
77 L.push('');
78 }
79 L.push('## Behaviours', '');
80 for (const area of AREAS) {
81 const bs = all.filter(b => b.area === area);
82 L.push(`### ${AREA_TITLES[area]} (\`${area}\`, ${bs.length})`, '');
83 L.push('| id | status | evidence | ply ops | tests |', '|---|---|---|---|---:|');
84 for (const b of bs) {
85 const ev = EVIDENCE_KINDS.filter(k => kindsOf(b).has(k)).map(k => SHORT[k]).join(', ');
86 const st = b.status.tag === 'unsupported' && b.status.observed.length ? 'unsupported (observed)' : b.status.tag;
87 L.push(`| \`${b.id}\` | ${st} | ${ev || '—'} | ${b.ops.map(o => `\`${o.symbol}\``).join(', ') || '—'} | ${b.tests.length} |`);
88 }
89 L.push('');
90 for (const b of bs) {
91 L.push(`#### \`${b.id}\``, '');
92 L.push(esc(b.description), '');
93 if (b.status.tag === 'unsupported') L.push(`Unsupported: ${esc(b.status.why)}`, '');
94 if (b.ops.length) L.push(`Ops: ${b.ops.map(o => `\`${o.module}\` \`${o.symbol}\``).join(', ')}.`, '');
95 if (b.tests.length) { L.push('Tests:', ''); for (const t of b.tests) L.push(`- \`${t.path}\`: ${esc(t.anchor)}`); L.push(''); }
96 const ev = evidenceOf(b);
97 if (ev.length) { L.push(b.status.tag === 'verified' ? 'Evidence:' : 'Observed:', ''); for (const e of ev) L.push(`- ${SHORT[e.kind]}: ${where(e)}`); L.push(''); }
98 for (const n of b.notes) L.push(`> ${esc(n)}`, '');
99 }
100 }
101 L.push('## Recorded runs', '');
102 L.push('| run | date | kinds | checks | harness |', '|---|---|---|---:|---|');
103 for (const [id, r] of Object.entries(runs)) L.push(`| \`${id}\` | ${r.date} | ${r.kinds.map(k => SHORT[k]).join(', ')} | ${r.checks.length} | ${esc(r.harness)} |`);
104 L.push('');
105 return `${L.join('\n')}\n`;
106}
108export function report(all = BEHAVIOURS) {
109 const c = counts(all);
110 console.log(`behaviours ${c.total}: verified ${c.verified}, implemented ${c.implemented}, unsupported ${c.unsupported}`);
111 for (const a of c.byArea) console.log(` ${a.area.padEnd(12)} ${String(a.total).padStart(3)} verified ${a.verified}, implemented ${a.implemented}, unsupported ${a.unsupported}`);
112 console.log(`verified by evidence kind: ${EVIDENCE_KINDS.map(k => `${k} ${c.byKind[k]}`).join(', ')}; any Artifacts evidence ${c.anyArtifacts}`);
113 for (const g of GAPS) console.log(`gap ${g.key}: ${c.gaps[g.key]}`);
114}
116if (process.argv[1] === fileURLToPath(import.meta.url)) {
117 if (process.argv.includes('--md')) {
118 writeFileSync(join(ROOT, 'COVERAGE.md'), renderMd());
119 console.log('wrote COVERAGE.md');
120 } else report();
121}
Diagnostics4 git calls · 450 ms
CallStatusSizeTime0450 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=445 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2003.0 KiB445 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3114 git.response.cache=MISS git.response.duration_ms=445 git.fetch.wants=9ab976ec5f33b6401e9643517c1b7ec15e1a0690 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3114 git.pkt.payload.max=3093 git.sideband.band1.bytes=3092 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e27eccbf9a0555
- Request
- 6a6b3d82…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format