// Both 2026-11-10 harness faults, replayed: the pin straggler (run 2026-11-10T20-48-18-470Z, flags-rules served by // 480225db 9.9 s after the 0% write) and the 53.7 s client stall (run 2026-11-10T20-54-06-051Z, 58 probes issued). import assert from 'node:assert/strict' import { readFileSync } from 'node:fs' import { test } from 'node:test' import { none, some } from '../src/option.ts' import { ClientHealth, gapStall, NO_STALLS, type Stall, type StallSource } from '../scripts/cd/client-health.ts' import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts' import { classifyProbe, judgeProbeWindow, type ProbeJudge, type ProbeOutcome } from '../scripts/cd/probe.ts' import { runProbeSchedule, type ProbeClock } from '../scripts/cd/probe-schedule.ts' import { runRamp, type Observation } from '../scripts/cd/ramp.ts' import { confirmRoute, coloOf, failureFault, stragglers, sustainedFrom, type RouteIo, type RouteSample } from '../scripts/cd/route-confirm.ts' import { liveVerdict } from '../scripts/cd/pipeline-plan.ts' import { liveRoute } from '../scripts/cd/pipeline.ts' import { pinReady } from '../scripts/cd/stage-deploy.ts' import { findRemote, harnessEvent, reconfirmRoute, reissuable } from '../scripts/cd/stage-smoke.ts' import type { VersionStats } from '../scripts/cd/telemetry.ts' const OLD = '480225db' const NEW = 'c8403e57' // ---- client health ---- test('gapStall: a 100 ms tick is healthy, a wall or monotonic jump over the bound is a stall', () => { assert.equal(gapStall(0, 0, 100, 100, 1000), null) assert.equal(gapStall(0, 0, 1000, 1000, 1000), null) assert.deepEqual(gapStall(10, 0, 53_743, 100, 1000), { from: 10, to: 53_743, ms: 53_733 }) // the guest monotonic clock jumps while the wall clock was stepped back by NTP: still a stall assert.deepEqual(gapStall(0, 0, 50, 5000, 1000), { from: 0, to: 5000, ms: 5000 }) }) test('ClientHealth records a stall and answers during / stalledMs by overlap', () => { let wall = 1_000 let mono = 0 const health = new ClientHealth({ wall: () => wall, mono: () => mono }, 1000, 100) wall += 100 mono += 100 assert.equal(health.check(), null) wall += 53_733 mono += 53_733 const stall = health.check() assert.deepEqual(stall, { from: 1_100, to: 54_833, ms: 53_733 }) assert.equal(health.during(0, 1_000).length, 0) assert.equal(health.during(50_000, 60_000).length, 1) assert.equal(health.stalledMs(50_000, 60_000), 4_833) assert.equal(health.stalledMs(0, 100_000), 53_733) assert.equal(health.stalls().length, 1) }) // ---- pin / route confirmation ---- function sample(at: number, round: number, version: string | null, status = 200): RouteSample { return { at, round, status, version, colo: 'IAD' } } test('sustainedFrom restarts at the round after a straggler', () => { const samples = [sample(1000, 1, NEW), sample(2000, 2, OLD), sample(2050, 2, NEW), sample(3000, 3, NEW), sample(4000, 4, NEW)] assert.equal(sustainedFrom(samples, NEW), 3000) assert.equal(sustainedFrom([...samples, sample(5000, 5, NEW, 0)], NEW), null) assert.equal(stragglers(samples, NEW).length, 1) assert.equal(coloOf('a587ec564e0da476-IAD'), 'IAD') assert.equal(coloOf(null), null) }) test('sustainedFrom judges rounds in round order, whatever order the samples arrived in', () => { const samples = [sample(3000, 3, NEW), sample(1000, 1, NEW), sample(2000, 2, OLD), sample(4000, 4, NEW), sample(3100, 3, NEW)] assert.equal(sustainedFrom(samples, NEW), 3000) assert.equal(sustainedFrom([], NEW), null) }) test('findRemote (#147): the first Artifacts remote anywhere in a cf answer is some, an answer with none is none', () => { const remote = 'https://abc123.artifacts.cloudflare.net/git/ns/repo.git' assert.deepEqual(findRemote({ result: { repo: { name: 'r', urls: ['https://example.com/x', remote] } } }), some(remote)) assert.deepEqual(findRemote([{ remote: 'git@example.com:x' }, { nested: [remote] }]), some(remote)) assert.deepEqual(findRemote({ result: { remote: '' } }), none()) assert.deepEqual(findRemote(null), none()) assert.deepEqual(findRemote('https://abc123.artifacts.cloudflare.net/other/'), none()) }) /** A fake edge: requests before `cutoverMs` after the write sometimes reach a machine without the new deployment. */ function edge(cutoverMs: number, version: (at: number, n: number) => string | null, status = 200): RouteIo & { calls: number } { let now = 0 let n = 0 const io = { calls: 0, now: () => now, sleep: async (ms: number) => { now += ms }, get: async () => { n += 1 io.calls += 1 now += 1 return { status, version: version(now, n), ray: `ray${n}-${n % 2 === 0 ? 'IAD' : 'EWR'}` } }, } void cutoverMs return io } const TUNABLES = { batch: 10, intervalMs: 1000, sustainMs: 10_000, maxMs: 120_000, requestMs: 2000 } test('replay 20-48-18: the old five-in-a-row check passed at +5 s, the straggler at +9.9 s; the sustained check confirms only after it', async () => { // one old answer in a hundred until +9.9 s, exactly the shape the 2026-11-10 run saw const io = edge(9_900, (at, n) => (at <= 9_900 && n % 37 === 0 ? OLD : NEW)) const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES) assert.equal(verdict.kind, 'confirmed') if (verdict.kind !== 'confirmed') return const late = stragglers(verdict.samples, NEW) assert.ok(late.length > 0) const lastLate = Math.max(...late.map(item => item.at)) assert.ok(verdict.sustainedFrom > lastLate, `sustained window ${verdict.sustainedFrom} must start after the last straggler ${lastLate}`) assert.ok(verdict.at - verdict.sustainedFrom >= TUNABLES.sustainMs) assert.ok(verdict.samples.length >= 100) }) test('a version that never settles is routing (harness), not a product failure', async () => { const io = edge(Infinity, (_at, n) => (n % 50 === 0 ? OLD : NEW)) const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES) assert.equal(verdict.kind, 'routing') if (verdict.kind === 'routing') assert.match(verdict.reason, /^routing: /) }) test('answers with no x-ply-version at all are the Worker', async () => { const io = edge(0, () => null) const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES) assert.equal(verdict.kind, 'worker') }) test('lost answers never confirm a route', async () => { const io = edge(0, () => null, 0) const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES) assert.equal(verdict.kind, 'routing') }) // ply-web#146: the repro from the issue (every GET answers HTTP 500 with the wanted x-ply-version). A served 5xx is the // product, never routing, and it is decided on the first such answer, not after maxMs. const PIN_HEADERS = { 'Cloudflare-Workers-Version-Overrides': `ply="${NEW}"` } test('#146 pin: every answer HTTP 500 from the wanted version is the Worker (served_5xx), at once', async () => { const io = edge(0, () => NEW, 500) const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: PIN_HEADERS, since: 0 }, io, TUNABLES) assert.equal(verdict.kind, 'worker') if (verdict.kind === 'worker') assert.equal(verdict.cause, 'served_5xx') assert.equal(io.calls, TUNABLES.batch, 'decided after the first round, not after maxMs') assert.equal(failureFault(verdict), 'worker') }) test('#146 one intermittent 5xx from the wanted version among 2xx answers is the Worker', async () => { let n = 0 let now = 0 const io: RouteIo = { now: () => now, sleep: async ms => { now += ms }, get: async () => { n += 1 now += 1 return { status: n === 57 ? 503 : 200, version: NEW, ray: `r${n}-IAD` } }, } const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES) assert.equal(verdict.kind, 'worker') if (verdict.kind === 'worker') assert.equal(verdict.cause, 'served_5xx') }) test('#146 a 5xx from another version is still routing, and other-version answers mixed with 2xx from the wanted version are routing', async () => { const mixed = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, edge(Infinity, (_at, n) => (n % 50 === 0 ? OLD : NEW)), TUNABLES) assert.equal(mixed.kind, 'routing') assert.equal(failureFault(mixed), 'harness_fault') const old5xx = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, edge(0, () => OLD, 500), TUNABLES) assert.equal(old5xx.kind, 'routing') }) test('#146 confirmed has no fault; headerless answers are the Worker (no_version_header)', async () => { const ok = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, edge(0, () => NEW), TUNABLES) assert.equal(ok.kind, 'confirmed') const bare = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, edge(0, () => null), TUNABLES) assert.equal(bare.kind, 'worker') if (bare.kind === 'worker') assert.equal(bare.cause, 'no_version_header') }) // ply-web#146 through the three callers that turn a verdict into a pipeline fault: the pin gate (stage-deploy pinReady), // the live check (pipeline liveRoute, judged by liveVerdict) and the smoke restart (stage-smoke reconfirmRoute). Each // fake records the headers it was sent, so the pin path is shown to carry the override and the live path to carry none. function recording(status: number, version: (n: number) => string | null): RouteIo & { readonly sent: Record[] } { let now = 0 let n = 0 const sent: Record[] = [] return { sent, now: () => now, sleep: async ms => { now += ms }, get: async (_url, headers) => { n += 1 now += 1 sent.push(headers) return { status, version: version(n), ray: `r${n}-IAD` } }, } } const SITE = { baseUrl: 'https://x', worker: 'ply' } const OVERRIDE = { 'Cloudflare-Workers-Version-Overrides': `ply="${NEW}"` } test('#146 pin path: pinReady fails with fault worker when the canary serves HTTP 500 on the override', async () => { const io = recording(500, () => NEW) const result = await pinReady(SITE, NEW, 0, io, TUNABLES) assert.deepEqual(io.sent[0], OVERRIDE) assert.equal(result.ok, false) if (result.ok) return assert.equal(result.fault, 'worker') assert.match(result.reason, /served HTTP 500/) assert.equal(io.sent.length, TUNABLES.batch, 'decided after the first round') }) test('#146 live path: liveRoute returns a Fail with fault worker, and liveVerdict rolls back', async () => { const io = recording(500, () => NEW) const lines: string[] = [] const result = await liveRoute(SITE, { version: NEW, log: line => lines.push(line) }, io, TUNABLES) assert.deepEqual(io.sent[0], {}) assert.notEqual(result, null) if (result === null) return assert.equal(result.fault, 'worker') assert.equal(result.stage, 'smoke') assert.equal(liveVerdict([result]), 'rollback') assert.match(lines.join('\n'), /route-confirm live: worker/) }) test('#146 live path: other-version answers mixed with 2xx from the wanted version halt (harness_fault), never roll back', async () => { const io = recording(200, n => (n % 50 === 0 ? OLD : NEW)) const result = await liveRoute(SITE, { version: NEW, log: () => {} }, io, TUNABLES) assert.notEqual(result, null) if (result === null) return assert.equal(result.fault, 'harness_fault') assert.equal(liveVerdict([result]), 'halt') }) test('#146 live path: a confirmed route is no failure', async () => { assert.equal(await liveRoute(SITE, { version: NEW, log: () => {} }, recording(200, () => NEW), TUNABLES), null) }) test('#146 smoke restart: reconfirmRoute fails with fault worker on a served 5xx, pinned and unpinned', async () => { const pinned = recording(503, () => NEW) const pin = await reconfirmRoute(SITE, NEW, true, pinned, TUNABLES) assert.deepEqual(pinned.sent[0], OVERRIDE) assert.equal(pin.ok ? null : pin.fault, 'worker') const unpinned = recording(503, () => NEW) const live = await reconfirmRoute(SITE, NEW, false, unpinned, TUNABLES) assert.deepEqual(unpinned.sent[0], {}) assert.equal(live.ok ? null : live.fault, 'worker') }) // The rule is status >= 500 (the issue). A 4xx from the wanted version is not a served 5xx: it never confirms (sustainedFrom // needs 2xx/3xx), so it ends as routing at maxMs, not as an immediate Worker verdict. for (const status of [404, 499]) { test(`#146 boundary: HTTP ${status} from the wanted version is not served_5xx`, async () => { const io = edge(0, () => NEW, status) const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES) assert.equal(verdict.kind, 'routing') assert.ok(io.calls > TUNABLES.batch, 'not decided on the first round') }) } // ---- smoke row attribution ---- function stalls(list: readonly Stall[]): StallSource { const overlap = (stall: Stall, from: number, to: number) => Math.max(0, Math.min(stall.to, to) - Math.max(stall.from, from)) return { check: () => null, during: (from, to) => list.filter(stall => overlap(stall, from, to) > 0), stalledMs: (from, to) => list.reduce((sum, stall) => sum + overlap(stall, from, to), 0), } } test('a smoke row served by another version is a routing event, never a pass or a product failure', () => { const event = harnessEvent({ started: 0, ended: 50, served: OLD, status: 200 }, NEW, NO_STALLS) assert.equal(event && event.kind, 'wrong_version') assert.equal(harnessEvent({ started: 0, ended: 50, served: NEW, status: 200 }, NEW, NO_STALLS), null) // no header at all is the Worker's to answer for (judged by the row), not routing assert.equal(harnessEvent({ started: 0, ended: 50, served: null, status: 200 }, NEW, NO_STALLS), null) // the 2026-11-10 replay: flags-rules answered by 480225db const replay = harnessEvent({ started: 9_800, ended: 9_900, served: OLD, status: 200 }, NEW, NO_STALLS) assert.ok(replay && replay.kind === 'wrong_version' && replay.served === OLD) }) test('a lost smoke row across a client stall is a client_stall event; without a stall it stays a loss', () => { const source = stalls([{ from: 1_000, to: 54_733, ms: 53_733 }]) const event = harnessEvent({ started: 500, ended: 55_000, served: null, status: 0 }, NEW, source) assert.ok(event && event.kind === 'client_stall' && event.stalledMs === 53_733) assert.equal(harnessEvent({ started: 60_000, ended: 62_000, served: null, status: 0 }, NEW, source), null) }) test('only GETs are re-issued; a write restarts the lifecycle', () => { assert.equal(reissuable({ method: 'GET' }), true) assert.equal(reissuable({ method: 'POST' }), false) assert.equal(reissuable({ method: 'DELETE' }), false) }) // ---- probe schedule extension ---- function policy(): RolloutPolicy { const raw = JSON.parse(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')) as Record raw.ingestPollMs = 0 const parsed = parsePolicy(JSON.stringify(raw)).policy return { ...parsed, steps: [{ percent: 100, dwellMs: 40_000 }] } } function judge(): ProbeJudge { const rollout = policy() return { maxLossRate: rollout.probe.maxLossRate, respondedMin: rollout.burn.minSamples, minSamples: rollout.probe.minSamples, stallLatencyMs: rollout.probe.stallLatencyMs, fivexxHard: rollout.gates.fivexx === 'hard', } } function answered(id: string, version: string, latencyMs: number, status = 200): ProbeOutcome { return classifyProbe(id, { kind: 'response', status, ray: some('ray'), latencyMs, version: some(version) }) } /** * A process that freezes for `pauseMs` at `pauseAt`: the clock jumps, the monitor (driven by the same clock) sees it, * and the probes in flight across it never answer. Probes issued otherwise answer after `latencyMs`. */ function frozenBox(pauseAt: number, pauseMs: number, latencyMs: number) { let at = 0 let paused = false const seen: Stall[] = [] let last = 0 const health: StallSource = { check: () => { if (at - last > 1000) { const stall = { from: last, to: at, ms: at - last } seen.push(stall) last = at return stall } last = at return null }, during: (from, to) => seen.filter(stall => Math.min(stall.to, to) - Math.max(stall.from, from) > 0), stalledMs: (from, to) => seen.reduce((sum, stall) => sum + Math.max(0, Math.min(stall.to, to) - Math.max(stall.from, from)), 0), } const waiting: { due: number; resolve: () => void }[] = [] const clock: ProbeClock = { now: () => at, sleep: async ms => { // the monitor ticks every 100 ms while the process runs; a freeze is one jump it sees afterwards let left = ms while (left > 0) { if (!paused && at + Math.min(left, 100) > pauseAt) { paused = true at += pauseMs } const step = Math.min(left, 100) at += step left -= step health.check() } for (const item of waiting.splice(0)) { if (item.due <= at) item.resolve() else waiting.push(item) } await Promise.resolve() }, } const issue = (seq: number) => { const issuedAt = at const crosses = !paused && issuedAt + latencyMs > pauseAt const promise = new Promise(resolve => { // its client timeout fires as soon as the process runs again: a timed_out loss, unless the stall is seen if (crosses) { waiting.push({ due: issuedAt + 2000, resolve: () => resolve({ kind: 'lost', id: `p-${seq}`, reason: 'timed_out' }) }) return } waiting.push({ due: issuedAt + latencyMs, resolve: () => resolve(answered(`p-${seq}`, NEW, latencyMs)) }) }) return { id: `p-${seq}`, promise } } return { clock, health, issue, read: () => at, seen } } test('replay 20-54-06: a 53.7 s freeze mid-dwell extends the window to minSamples real samples and nothing is lost', async () => { const rollout = policy() const box = frozenBox(5_000, 53_733, 50) const outcomes = await runProbeSchedule({ clock: box.clock, dwellMs: 40_000, periodMs: 1000 / rollout.probe.rps, maxInFlight: rollout.probe.maxInFlight, batch: 2, graceMs: rollout.probe.timeoutMs, health: box.health, extension: { maxMs: 120_000, minReal: rollout.probe.minSamples }, issue: box.issue, shed: seq => ({ kind: 'shed', id: `shed-${seq}` }), stop: () => {}, }) const window = judgeProbeWindow(outcomes, judge()) assert.equal(box.seen.length, 1) assert.ok(window.loss.stalled >= 1) assert.equal(window.loss.count, 0, 'probes in flight across the freeze are stalled, not lost') assert.equal(window.loss.shed, 0, 'missed ticks are skipped, not burst into sheds') assert.ok(window.loss.total >= rollout.probe.minSamples, `${window.loss.total} real samples`) assert.equal(window.kind, 'ok') // the dwell ran its 40 s of healthy time plus the freeze, not more than the cap assert.ok(box.read() >= 40_000 + 53_733 && box.read() <= 40_000 + 120_000 + rollout.probe.timeoutMs + 1000) }) test('a freeze longer than the extension cap is harness_unhealthy, never a product verdict', async () => { const rollout = policy() const box = frozenBox(5_000, 400_000, 50) const outcomes = await runProbeSchedule({ clock: box.clock, dwellMs: 40_000, periodMs: 1000 / rollout.probe.rps, maxInFlight: rollout.probe.maxInFlight, batch: 2, graceMs: rollout.probe.timeoutMs, health: box.health, extension: { maxMs: 120_000, minReal: rollout.probe.minSamples }, issue: box.issue, shed: seq => ({ kind: 'shed', id: `shed-${seq}` }), stop: () => {}, }) const window = judgeProbeWindow(outcomes, judge()) assert.equal(window.kind, 'harness_unhealthy') assert.match(window.kind === 'harness_unhealthy' ? window.reason : '', /^harness unhealthy: /) }) test('without a stall the dwell is not extended', async () => { const rollout = policy() const box = frozenBox(Infinity, 0, 50) await runProbeSchedule({ clock: box.clock, dwellMs: 10_000, periodMs: 1000 / rollout.probe.rps, maxInFlight: rollout.probe.maxInFlight, batch: 2, graceMs: rollout.probe.timeoutMs, health: box.health, extension: { maxMs: 120_000, minReal: rollout.probe.minSamples }, issue: box.issue, shed: seq => ({ kind: 'shed', id: `shed-${seq}` }), stop: () => {}, }) assert.equal(box.seen.length, 0) // the dwell plus at most the in-flight grace, never the 120 s extension assert.ok(box.read() <= 10_000 + rollout.probe.timeoutMs + 100, String(box.read())) }) test('judge: stalled probes leave the denominator; a served 5xx still rolls back', () => { const outcomes: ProbeOutcome[] = [] for (let index = 0; index < 53; index += 1) outcomes.push(answered(`ok-${index}`, NEW, 50)) for (let index = 0; index < 5; index += 1) outcomes.push({ kind: 'stalled', id: `s-${index}` }) const unhealthy = judgeProbeWindow(outcomes, judge()) assert.equal(unhealthy.kind, 'harness_unhealthy') assert.equal(unhealthy.loss.total, 53) assert.equal(unhealthy.loss.count, 0) const withFive = [...outcomes, answered('five', NEW, 50, 500)] assert.equal(judgeProbeWindow(withFive, judge()).kind, 'rollback') const enough: ProbeOutcome[] = [] for (let index = 0; index < judge().minSamples; index += 1) enough.push(answered(`ok-${index}`, NEW, 50)) enough.push({ kind: 'stalled', id: 'client-stall' }) assert.equal(judgeProbeWindow(enough, judge()).kind, 'ok') }) function stats(requests: number): VersionStats { return { requests, invocations: requests, exceptions: 0, errors: 0, fivexx: 0, errorLogs: 0, wallMs: [10, 10, 10, 10, 10], cpuMs: [10], fetchMs: [10], v1: [], internal: [], } } function unhealthyRun(rollout: RolloutPolicy) { const counts = { rolled: 0, splits: [] as number[], dwells: 0 } const outcomes: ProbeOutcome[] = [] for (let index = 0; index < 53; index += 1) outcomes.push(answered(`ok-${index}`, NEW, 50)) outcomes.push({ kind: 'stalled', id: 'client-stall 0-53733 53733ms' }) const run = runRamp(rollout, { now: () => 0, sleep: async () => {}, dwell: async () => { counts.dwells += 1 return { outcomes } }, applySplit: async (percent: number) => { counts.splits.push(percent) return null }, observe: async (): Promise => { const next = stats(200) return { kind: 'sample', next, previous: stats(0), aggregate: next, landed: 200 } }, rollback: async () => { counts.rolled += 1 return null }, }) return { run, counts } } // ply-web#145: the step's split write already moved traffic, so an unjudged step must settle it like a harness fault. test('runRamp: an unhealthy client above 0% rolls back under the checked-in onHarnessFault policy and says so', async () => { const rollout = { ...policy(), steps: [{ percent: 10, dwellMs: 40_000 }] } assert.equal(rollout.probe.onHarnessFault, 'telemetry') const { run, counts } = unhealthyRun(rollout) const result = await run assert.equal(counts.rolled, 1) assert.deepEqual(counts.splits, [10]) assert.equal(counts.dwells, 1) assert.equal(result.state.kind, 'held') if (result.state.kind === 'held') { assert.equal(result.state.outcome, 'harness_unhealthy') assert.equal('traffic' in result.state ? result.state.traffic : null, 'rolled_back') assert.doesNotMatch(result.state.reason, /prod untouched/) assert.match(result.state.reason, /^harness unhealthy: /) assert.match(result.state.reason, /rolled back to previous$/) } }) test('runRamp: an unhealthy client under onHarnessFault hold with no passed step rolls back', async () => { const base = policy() const rollout = { ...base, steps: [{ percent: 10, dwellMs: 40_000 }], probe: { ...base.probe, onHarnessFault: 'hold' as const } } const { run, counts } = unhealthyRun(rollout) const result = await run assert.equal(counts.rolled, 1) assert.deepEqual(counts.splits, [10]) assert.equal(result.state.kind === 'held' && 'traffic' in result.state ? result.state.traffic : null, 'rolled_back') })