// The Git behaviour coverage matrix, from the command line. Reads test/behaviours/matrix.ts (the behaviours) and // test/behaviours/runs.ts (the recorded runs they cite). // // npm run coverage-matrix counts by area, status and evidence kind, then the gaps // npm run coverage-matrix -- --md regenerate COVERAGE.md (test/behaviours/matrix.test.js fails if it drifts) import { writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { BEHAVIOURS } from '../test/behaviours/matrix.ts'; import { RUNS } from '../test/behaviours/runs.ts'; import { AREAS, AREA_TITLES, EVIDENCE_KINDS } from '../test/behaviours/types.ts'; const ROOT = join(fileURLToPath(import.meta.url), '..', '..'); const STATUSES = ['verified', 'implemented', 'unsupported']; const ARTIFACTS_KINDS = ['ply_vs_artifacts', 'native_git_vs_artifacts', 'wire_probe_vs_artifacts']; const SHORT = { ply_vs_reference_git: 'ply↔git', ply_vs_artifacts: 'ply↔Artifacts', native_git_vs_artifacts: 'git↔Artifacts', wire_probe_vs_artifacts: 'wire↔Artifacts' }; const esc = s => String(s).replace(/\|/g, '\\|').replace(/\n/g, ' ').replace(/ b.status.tag === 'verified' ? b.status.evidence : b.status.tag === 'unsupported' ? b.status.observed : []; const kindsOf = b => new Set(evidenceOf(b).map(e => e.kind)); const where = e => e.ref.tag === 'test' ? `\`${e.ref.path}\`` : `${e.ref.run}: \`${esc(e.ref.check)}\``; // The gap classes, worst first. A behaviour is in the first class that holds for it. export const GAPS = [ { key: 'implemented', title: 'Exercised by ply, judged by nothing independent', holds: b => b.status.tag === 'implemented' }, { key: 'no-artifacts', title: 'Verified against git http-backend only: no evidence of any kind from Artifacts', holds: b => b.status.tag === 'verified' && !ARTIFACTS_KINDS.some(k => kindsOf(b).has(k)) }, { key: 'no-ply-on-artifacts', title: 'Artifacts observed by stock git or wire probes, never by ply', holds: b => b.status.tag === 'verified' && !kindsOf(b).has('ply_vs_artifacts') }, { key: 'no-reference', title: 'Verified on Artifacts, but ply\'s probe is not checked against git http-backend', holds: b => b.status.tag === 'verified' && !kindsOf(b).has('ply_vs_reference_git') }, { key: 'unsupported', title: 'Not exercisable by ply', holds: b => b.status.tag === 'unsupported' }, ]; export const gapOf = b => GAPS.find(g => g.holds(b)); export function counts(all = BEHAVIOURS) { const byArea = AREAS.map(area => { const bs = all.filter(b => b.area === area); return { area, total: bs.length, ...Object.fromEntries(STATUSES.map(s => [s, bs.filter(b => b.status.tag === s).length])) }; }); const verified = all.filter(b => b.status.tag === 'verified'); const byKind = Object.fromEntries(EVIDENCE_KINDS.map(k => [k, verified.filter(b => kindsOf(b).has(k)).length])); const anyArtifacts = verified.filter(b => ARTIFACTS_KINDS.some(k => kindsOf(b).has(k))).length; const gaps = Object.fromEntries(GAPS.map(g => [g.key, all.filter(b => gapOf(b) === g).length])); return { total: all.length, byArea, byKind, anyArtifacts, gaps, ...Object.fromEntries(STATUSES.map(s => [s, all.filter(b => b.status.tag === s).length])) }; } export function renderMd(all = BEHAVIOURS, runs = RUNS) { const c = counts(all); const L = []; L.push('# Git behaviour coverage', ''); L.push('', ''); L.push('ply is a Git protocol conformance and behaviour-testing instrument for Cloudflare Artifacts as well as a client. This matrix counts **server behaviours**, not git commands: each row is a behaviour or failure mode of a Git server, the ply operations and tests that exercise it, and who has independently checked the server\'s answer.', ''); L.push('- **unsupported**: ply has no operation that exercises it. Stock git or hand-built wire probes may still have observed Artifacts.'); L.push('- **implemented**: ply exercises it, but only against fakes or its own expectations; nothing independent has judged the server.'); L.push('- **verified**: at least one independent check. The types make a verified row without evidence impossible.', ''); L.push('Evidence kinds:', ''); L.push('- `ply_vs_reference_git` (ply↔git): ply against git http-backend in this suite, with git\'s own output as the expected value; the cited file runs real git.'); L.push('- `ply_vs_artifacts` (ply↔Artifacts): ply against Artifacts in a recorded run.'); L.push('- `native_git_vs_artifacts` (git↔Artifacts): stock git against Artifacts in a recorded run, with git http-backend beside it as the baseline.'); L.push('- `wire_probe_vs_artifacts` (wire↔Artifacts): hand-built requests (pkt-lines, packs, HTTP framing) against Artifacts beside the same baseline. Neither ply nor stock git sends these bytes.', ''); L.push('Only Artifacts evidence says anything about Artifacts; reference evidence says ply\'s probe is right. Recorded runs are private (their harnesses hold credentials): `test/behaviours/runs.ts` keeps each run\'s id, date and check names, never its verdicts.', ''); L.push('## Summary', ''); L.push(`**${c.total}** behaviours: **${c.verified}** verified, **${c.implemented}** implemented, **${c.unsupported}** unsupported. Of the verified, **${c.anyArtifacts}** carry some Artifacts evidence and **${c.byKind.ply_vs_artifacts}** carry evidence of ply itself against Artifacts.`, ''); L.push('| area | behaviours | verified | implemented | unsupported |', '|---|---:|---:|---:|---:|'); for (const a of c.byArea) L.push(`| ${a.area} | ${a.total} | ${a.verified} | ${a.implemented} | ${a.unsupported} |`); L.push(`| **all** | **${c.total}** | **${c.verified}** | **${c.implemented}** | **${c.unsupported}** |`, ''); L.push('Verified behaviours by evidence kind (a behaviour counts once under each kind it has):', ''); L.push('| evidence kind | verified behaviours |', '|---|---:|'); for (const k of EVIDENCE_KINDS) L.push(`| \`${k}\` | ${c.byKind[k]} |`); L.push(''); L.push('## Gaps', ''); L.push('Each behaviour that is not verified by both ply↔git and ply↔Artifacts, filed under the first of these classes that holds:', ''); for (const g of GAPS) { const bs = all.filter(b => gapOf(b) === g); L.push(`### ${g.title} (${bs.length})`, ''); if (bs.length === 0) { L.push('None.', ''); continue; } for (const b of bs) L.push(`- \`${b.id}\`: ${esc(b.description)}`); L.push(''); } L.push('## Behaviours', ''); for (const area of AREAS) { const bs = all.filter(b => b.area === area); L.push(`### ${AREA_TITLES[area]} (\`${area}\`, ${bs.length})`, ''); L.push('| id | status | evidence | ply ops | tests |', '|---|---|---|---|---:|'); for (const b of bs) { const ev = EVIDENCE_KINDS.filter(k => kindsOf(b).has(k)).map(k => SHORT[k]).join(', '); const st = b.status.tag === 'unsupported' && b.status.observed.length ? 'unsupported (observed)' : b.status.tag; L.push(`| \`${b.id}\` | ${st} | ${ev || '—'} | ${b.ops.map(o => `\`${o.symbol}\``).join(', ') || '—'} | ${b.tests.length} |`); } L.push(''); for (const b of bs) { L.push(`#### \`${b.id}\``, ''); L.push(esc(b.description), ''); if (b.status.tag === 'unsupported') L.push(`Unsupported: ${esc(b.status.why)}`, ''); if (b.ops.length) L.push(`Ops: ${b.ops.map(o => `\`${o.module}\` \`${o.symbol}\``).join(', ')}.`, ''); if (b.tests.length) { L.push('Tests:', ''); for (const t of b.tests) L.push(`- \`${t.path}\`: ${esc(t.anchor)}`); L.push(''); } const ev = evidenceOf(b); if (ev.length) { L.push(b.status.tag === 'verified' ? 'Evidence:' : 'Observed:', ''); for (const e of ev) L.push(`- ${SHORT[e.kind]}: ${where(e)}`); L.push(''); } for (const n of b.notes) L.push(`> ${esc(n)}`, ''); } } L.push('## Recorded runs', ''); L.push('| run | date | kinds | checks | harness |', '|---|---|---|---:|---|'); for (const [id, r] of Object.entries(runs)) L.push(`| \`${id}\` | ${r.date} | ${r.kinds.map(k => SHORT[k]).join(', ')} | ${r.checks.length} | ${esc(r.harness)} |`); L.push(''); return `${L.join('\n')}\n`; } export function report(all = BEHAVIOURS) { const c = counts(all); console.log(`behaviours ${c.total}: verified ${c.verified}, implemented ${c.implemented}, unsupported ${c.unsupported}`); for (const a of c.byArea) console.log(` ${a.area.padEnd(12)} ${String(a.total).padStart(3)} verified ${a.verified}, implemented ${a.implemented}, unsupported ${a.unsupported}`); console.log(`verified by evidence kind: ${EVIDENCE_KINDS.map(k => `${k} ${c.byKind[k]}`).join(', ')}; any Artifacts evidence ${c.anyArtifacts}`); for (const g of GAPS) console.log(`gap ${g.key}: ${c.gaps[g.key]}`); } if (process.argv[1] === fileURLToPath(import.meta.url)) { if (process.argv.includes('--md')) { writeFileSync(join(ROOT, 'COVERAGE.md'), renderMd()); console.log('wrote COVERAGE.md'); } else report(); }