import assert from 'node:assert/strict' import { readFileSync } from 'node:fs' import { test } from 'node:test' import { FAST_BURN, MIN_SAMPLES } from '../scripts/cd/slo.ts' import { applyGates, type SmokeJudge } from '../scripts/cd/telemetry.ts' import { DEFAULT_MAX_LOSS_RATE, DEFAULT_POLICY_PATH, loadPolicy, parsePolicy, splitPolicy } from '../scripts/cd/policy.ts' function text(): string { return readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8') } function raw(): Record { return JSON.parse(text()) as Record } test('the checked-in policy is the correctness ramp', () => { const loaded = loadPolicy(DEFAULT_POLICY_PATH) assert.equal(loaded.sha256, parsePolicy(text()).sha256) assert.equal(loaded.policy.gates.latency.mode, 'informational') assert.equal(loaded.policy.gates.smoke, 'hard') assert.equal(loaded.policy.gates.fivexx, 'hard') assert.equal(loaded.policy.gates.exceptions, 'hard') assert.equal(loaded.policy.gates.errorLogs, 'hard') assert.equal(loaded.policy.gates.nonOk, 'hard') assert.equal(loaded.policy.gates.burn, 'hard') assert.equal(loaded.policy.ingestWaitMs, 20_000) assert.equal(loaded.policy.warmupMs, 10_000) assert.equal(loaded.policy.probe.rps, 25) assert.deepEqual(loaded.policy.probe.calls.map(call => call.path), [ '/', '/v1/discover?remote=https%3A%2F%2F5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net%2Fgit%2Fpublic-v0%2Fply.git', ]) assert.deepEqual(loaded.policy.steps.map(step => step.percent), [10, 25, 50, 75, 100]) assert.equal(loaded.policy.burn.fastBurn, FAST_BURN) assert.equal(loaded.policy.burn.minSamples, MIN_SAMPLES) assert.equal(loaded.policy.split.z, 3) assert.equal(loaded.policy.split.minNewRequests, MIN_SAMPLES) assert.equal(loaded.policy.split.onMismatch, 'rollback') assert.equal(loaded.policy.split.onInsufficient, 'hold') assert.equal(loaded.policy.probe.maxLossRate, 1 / 790) assert.equal(loaded.policy.probe.onHarnessFault, 'telemetry') assert.equal(loaded.policy.probe.calibration.source, 'run 2026-11-01T17-06-08-780Z') assert.equal(loaded.policy.probe.calibration.date, '2026-11-01') assert.equal(loaded.policy.probe.calibration.window, '100% ramp step') assert.equal(loaded.policy.probe.calibration.n, 790) assert.equal(loaded.policy.probe.calibration.lost, 1) assert.equal(loaded.policy.probe.calibration.web, '789ba93') assert.equal(loaded.policy.probe.calibration.ply, '1743461') assert.equal(loaded.policy.probe.calibration.observedRate, '1/790') assert.equal(loaded.policy.probe.timeoutMs, 2_000) assert.equal(loaded.policy.probe.maxInFlight, 50) assert.equal(loaded.policy.probe.minSamples, 1_000) assert.equal(loaded.policy.probe.stallLatencyMs, 1_000) assert.equal(loaded.policy.probe.windowAttempts, 2) assert.equal(loaded.policy.probe.windowBackoffMs, 1_000) assert.equal(loaded.policy.probe.schedule.source, 'run 2026-11-01T19-40-35-485Z') assert.equal(loaded.policy.probe.schedule.date, '2026-11-01') assert.equal(loaded.policy.probe.schedule.window, '50% ramp step') assert.equal(loaded.policy.probe.schedule.n, 128) assert.equal(loaded.policy.probe.schedule.responded, 118) assert.equal(loaded.policy.probe.schedule.lost, 10) assert.equal(loaded.policy.probe.schedule.healthyN, 1_990) assert.equal(loaded.policy.probe.schedule.web, '22bfb79') assert.equal(loaded.policy.probe.schedule.oldVersion, '4f3d7acf') assert.equal(loaded.policy.probe.schedule.newVersion, '19ee3cab') assert.equal(loaded.policy.probe.schedule.healthyP50Ms, '41-101') assert.equal(loaded.policy.probe.schedule.observedStallMs, 69_830) assert.equal(loaded.policy.probe.schedule.dwellMs, 40_000) assert.equal(loaded.policy.probe.schedule.observedRate, '10/128') assert.match(loaded.policy.probe.schedule.headroom, /PRIOR/) assert.ok(loaded.policy.probe.timeoutMs > 101 * 10) assert.ok(loaded.policy.probe.timeoutMs < loaded.policy.probe.schedule.observedStallMs) assert.ok(loaded.policy.probe.minSamples > loaded.policy.probe.schedule.n) assert.ok(loaded.policy.probe.minSamples < loaded.policy.probe.schedule.healthyN) assert.equal(loaded.policy.cleanup.attempts, 3) assert.equal(loaded.policy.cleanup.backoffMs, 1_000) assert.equal(loaded.policy.cleanup.staleMinutes, 60) assert.equal(loaded.policy.cleanup.calibration.source, 'run 2026-11-01T17-29-43-122Z') assert.equal(loaded.policy.cleanup.calibration.date, '2026-11-01') assert.equal(loaded.policy.cleanup.calibration.window, 'post-ramp scratch repo delete') assert.equal(loaded.policy.cleanup.calibration.n, 1) assert.equal(loaded.policy.cleanup.calibration.latencyMs, 103_000) assert.equal(loaded.policy.telemetry.query.attempts, 3) assert.equal(loaded.policy.telemetry.query.backoffMs, 1_000) assert.equal(loaded.policy.telemetry.query.calibration.source, 'run 2026-11-01T19-17-18-170Z') assert.equal(loaded.policy.telemetry.query.calibration.date, '2026-11-01') assert.equal(loaded.policy.telemetry.query.calibration.window, '100% ramp step') assert.equal(loaded.policy.telemetry.query.calibration.n, 1) assert.equal(loaded.policy.telemetry.query.calibration.web, '07568e0') assert.equal(loaded.policy.telemetry.query.calibration.ply, '1743461') assert.match(loaded.policy.telemetry.query.calibration.observed, /APIError timeout Envelope Error/) assert.match(loaded.policy.telemetry.query.calibration.headroom, /PRIOR/) assert.equal(loaded.policy.cleanup.calibration.web, '9558b10') assert.equal(loaded.policy.cleanup.calibration.namespace, 'ply-loadgen') assert.match(loaded.policy.cleanup.calibration.observed, /404 \[10200\] Repository not found/) assert.match(loaded.policy.cleanup.calibration.headroom, /PRIOR/) assert.deepEqual(loaded.policy.steps.map(step => step.dwellMs), [30_000, 25_000, 25_000, 25_000, 40_000]) }) test('a policy that cannot be run is rejected', () => { const decreasing = raw() const steps = decreasing.steps as { percent: number; dwellMs: number }[] steps[1] = { percent: 1, dwellMs: 40_000 } assert.throws(() => parsePolicy(JSON.stringify(decreasing)), /strictly increasing/) const early = raw() ;(early.steps as { percent: number }[]).pop() assert.throws(() => parsePolicy(JSON.stringify(early)), /last step must be 100/) const gate = raw() gate.gates = { ...(gate.gates as Record), latency: { mode: 'gate' } } assert.throws(() => parsePolicy(JSON.stringify(gate)), /latency=gate needs minSamples, p99Ratio, and effectFloorMs/) const extra = raw() extra.region = 'iad' assert.throws(() => parsePolicy(JSON.stringify(extra)), /unknown field/) assert.throws(() => parsePolicy('{'), /not JSON/) const mismatch = raw() mismatch.split = { ...(mismatch.split as Record), onMismatch: 'advance' } assert.throws(() => parsePolicy(JSON.stringify(mismatch)), /hold or rollback/) const thin = raw() thin.split = { ...(thin.split as Record), minNewRequests: 0 } assert.throws(() => parsePolicy(JSON.stringify(thin)), /minNewRequests/) const zee = raw() zee.split = { ...(zee.split as Record), z: 0 } assert.throws(() => parsePolicy(JSON.stringify(zee)), /split\.z/) const spaced = raw() const spacedCalls = (spaced.probe as { calls: { path: string }[] }).calls const discover = spacedCalls[1] if (!discover) throw new Error('missing discover call') discover.path = '/v1/discover?remote=a b' assert.throws(() => parsePolicy(JSON.stringify(spaced)), /probe\.calls\[1\]\.path/) const bare = raw() const bareCalls = (bare.probe as { calls: { path: string }[] }).calls const root = bareCalls[0] if (!root) throw new Error('missing root call') root.path = 'discover' assert.throws(() => parsePolicy(JSON.stringify(bare)), /must be a path starting with \//) }) test('the checked-in probe can see a correctness regression on the smallest step', () => { const loaded = loadPolicy(DEFAULT_POLICY_PATH).policy const dwellS = loaded.steps[0]?.dwellMs ?? 0 const requests = loaded.probe.rps * loaded.probe.calls.length * (dwellS / 1000) const open = loaded.steps.map(step => step.percent).filter(percent => percent < 100) const smallest = Math.min(...open) const expectedNew = requests * (smallest / 100) assert.equal(requests, 1500) assert.equal(smallest, 10) assert.equal(expectedNew, 150) assert.ok(expectedNew >= loaded.split.minNewRequests) const atFloor = 1 - 0.05 ** (1 / loaded.split.minNewRequests) const atExpected = 1 - 0.05 ** (1 / expectedNew) assert.ok(Math.abs(atFloor - 0.0295) < 0.001) assert.ok(Math.abs(atExpected - 0.0198) < 0.001) }) test('every checked-in step reaches its sample floors at the worst measured routed share, and cites its evidence', () => { const loaded = loadPolicy(DEFAULT_POLICY_PATH).policy // Worst new-version share per percent in the ledger (cd.policy.json calibration[steps[*].dwellMs]). const worst: Record = { 10: 0.0874, 25: 0.232, 50: 0.4765, 75: 0.734, 100: 0.98 } for (const step of loaded.steps) { const issued = loaded.probe.rps * loaded.probe.calls.length * (step.dwellMs / 1000) assert.ok(issued >= loaded.probe.minSamples, `${step.percent}%: ${issued} issued`) assert.ok(issued >= loaded.burn.minSamples) const share = worst[step.percent] if (share === Number(share)) assert.ok(issued * share >= loaded.split.minNewRequests, `${step.percent}%: ${issued * share} new`) } for (const field of ['steps[*].dwellMs', 'ingestPollMs', 'ingestWaitMs']) { assert.ok(field in loaded.calibration, `calibration for ${field}`) } const bad = { ...raw(), calibration: { 'promote.watchMs': { source: 's', date: 'd', window: 'w', n: 1, observed: 'o', headroom: 'h' } } } assert.throws(() => parsePolicy(JSON.stringify(bad)), /calibration\.promote\.watchMs: names no schedule field/) }) test('latency=gate with thresholds is a policy, and warm-up may be omitted', () => { const body = raw() body.gates = { ...(body.gates as Record), latency: { mode: 'gate', minSamples: 20, p99Ratio: 2, effectFloorMs: 10 }, } delete body.warmupMs const loaded = parsePolicy(JSON.stringify(body)) assert.equal(loaded.policy.gates.latency.mode, 'gate') assert.equal(loaded.policy.warmupMs, 0) if (loaded.policy.gates.latency.mode === 'gate') assert.equal(loaded.policy.gates.latency.p99Ratio, 2) }) test('--policy is taken off the command line', () => { assert.deepEqual(splitPolicy(['--policy', 'other.json', 'cd', '--no-bump']), { argv: ['cd', '--no-bump'], policyPath: 'other.json', }) assert.deepEqual(splitPolicy(['cd']), { argv: ['cd'], policyPath: null }) const missing = splitPolicy(['--policy']) assert.equal('error' in missing, true) const twice = splitPolicy(['--policy', 'a.json', '--policy', 'b.json']) assert.equal('error' in twice, true) }) test('informational latency does not fail smoke telemetry', () => { const verdict = { pass: true, total: 1, matched: 1, allowance: 2, lost: [], unconfirmed: [], otherVersion: [], reasons: [], line: 'telemetry: 1/1 matched', } const judge: SmokeJudge = { kind: 'worker', line: 'telemetry: 1/1 matched; latency', failures: [{ kind: 'latency', metric: 'wall', p: 'p99', value: 30_000, ceiling: 25_000 }], verdict, } const gates = loadPolicy(DEFAULT_POLICY_PATH).policy.gates const applied = applyGates(judge, gates) assert.equal(applied.kind, 'pass') if (applied.kind === 'pass') assert.match(applied.line, /informational:/) const gated = applyGates(judge, { ...gates, latency: { mode: 'gate', minSamples: 20, p99Ratio: 2, effectFloorMs: 10 }, }) assert.equal(gated.kind, 'worker') }) test('telemetry.maxLossRate is validated and defaults when omitted', () => { const base = JSON.parse(text()) as Record assert.equal(parsePolicy(text()).policy.telemetry.maxLossRate, DEFAULT_MAX_LOSS_RATE) const { telemetry: _omitted, ...without } = base assert.equal(parsePolicy(JSON.stringify(without)).policy.telemetry.maxLossRate, DEFAULT_MAX_LOSS_RATE) const tight = 0.1 const query = (base.telemetry as { query: unknown }).query assert.equal(parsePolicy(JSON.stringify({ ...base, telemetry: { maxLossRate: tight, query } })).policy.telemetry.maxLossRate, tight) for (const bad of [0, -0.1, 1.5, '0.5', null]) { assert.throws(() => parsePolicy(JSON.stringify({ ...base, telemetry: { maxLossRate: bad, query } })), /telemetry.maxLossRate: must be a number in \(0, 1\]/) } assert.throws(() => parsePolicy(JSON.stringify({ ...base, telemetry: { maxLoss: 1 } })), /telemetry.maxLoss: unknown field/) })