main
1// Both 2026-11-10 harness faults, replayed: the pin straggler (run 2026-11-10T20-48-18-470Z, flags-rules served by
2// 480225db 9.9 s after the 0% write) and the 53.7 s client stall (run 2026-11-10T20-54-06-051Z, 58 probes issued).
3import assert from 'node:assert/strict'
4import { readFileSync } from 'node:fs'
5import { test } from 'node:test'
6import { none, some } from '../src/option.ts'
7import { ClientHealth, gapStall, NO_STALLS, type Stall, type StallSource } from '../scripts/cd/client-health.ts'
8import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
9import { classifyProbe, judgeProbeWindow, type ProbeJudge, type ProbeOutcome } from '../scripts/cd/probe.ts'
10import { runProbeSchedule, type ProbeClock } from '../scripts/cd/probe-schedule.ts'
11import { runRamp, type Observation } from '../scripts/cd/ramp.ts'
12import { confirmRoute, coloOf, failureFault, stragglers, sustainedFrom, type RouteIo, type RouteSample } from '../scripts/cd/route-confirm.ts'
13import { liveVerdict } from '../scripts/cd/pipeline-plan.ts'
14import { liveRoute } from '../scripts/cd/pipeline.ts'
15import { pinReady } from '../scripts/cd/stage-deploy.ts'
16import { findRemote, harnessEvent, reconfirmRoute, reissuable } from '../scripts/cd/stage-smoke.ts'
17import type { VersionStats } from '../scripts/cd/telemetry.ts'
19const OLD = '480225db'
20const NEW = 'c8403e57'
22// ---- client health ----
24test('gapStall: a 100 ms tick is healthy, a wall or monotonic jump over the bound is a stall', () => {
25 assert.equal(gapStall(0, 0, 100, 100, 1000), null)
26 assert.equal(gapStall(0, 0, 1000, 1000, 1000), null)
27 assert.deepEqual(gapStall(10, 0, 53_743, 100, 1000), { from: 10, to: 53_743, ms: 53_733 })
28 // the guest monotonic clock jumps while the wall clock was stepped back by NTP: still a stall
29 assert.deepEqual(gapStall(0, 0, 50, 5000, 1000), { from: 0, to: 5000, ms: 5000 })
30})
32test('ClientHealth records a stall and answers during / stalledMs by overlap', () => {
33 let wall = 1_000
34 let mono = 0
35 const health = new ClientHealth({ wall: () => wall, mono: () => mono }, 1000, 100)
36 wall += 100
37 mono += 100
38 assert.equal(health.check(), null)
39 wall += 53_733
40 mono += 53_733
41 const stall = health.check()
42 assert.deepEqual(stall, { from: 1_100, to: 54_833, ms: 53_733 })
43 assert.equal(health.during(0, 1_000).length, 0)
44 assert.equal(health.during(50_000, 60_000).length, 1)
45 assert.equal(health.stalledMs(50_000, 60_000), 4_833)
46 assert.equal(health.stalledMs(0, 100_000), 53_733)
47 assert.equal(health.stalls().length, 1)
48})
50// ---- pin / route confirmation ----
52function sample(at: number, round: number, version: string | null, status = 200): RouteSample {
53 return { at, round, status, version, colo: 'IAD' }
54}
56test('sustainedFrom restarts at the round after a straggler', () => {
57 const samples = [sample(1000, 1, NEW), sample(2000, 2, OLD), sample(2050, 2, NEW), sample(3000, 3, NEW), sample(4000, 4, NEW)]
58 assert.equal(sustainedFrom(samples, NEW), 3000)
59 assert.equal(sustainedFrom([...samples, sample(5000, 5, NEW, 0)], NEW), null)
60 assert.equal(stragglers(samples, NEW).length, 1)
61 assert.equal(coloOf('a587ec564e0da476-IAD'), 'IAD')
62 assert.equal(coloOf(null), null)
63})
65test('sustainedFrom judges rounds in round order, whatever order the samples arrived in', () => {
66 const samples = [sample(3000, 3, NEW), sample(1000, 1, NEW), sample(2000, 2, OLD), sample(4000, 4, NEW), sample(3100, 3, NEW)]
67 assert.equal(sustainedFrom(samples, NEW), 3000)
68 assert.equal(sustainedFrom([], NEW), null)
69})
71test('findRemote (#147): the first Artifacts remote anywhere in a cf answer is some, an answer with none is none', () => {
72 const remote = 'https://abc123.artifacts.cloudflare.net/git/ns/repo.git'
73 assert.deepEqual(findRemote({ result: { repo: { name: 'r', urls: ['https://example.com/x', remote] } } }), some(remote))
74 assert.deepEqual(findRemote([{ remote: 'git@example.com:x' }, { nested: [remote] }]), some(remote))
75 assert.deepEqual(findRemote({ result: { remote: '' } }), none())
76 assert.deepEqual(findRemote(null), none())
77 assert.deepEqual(findRemote('https://abc123.artifacts.cloudflare.net/other/'), none())
78})
80/** A fake edge: requests before `cutoverMs` after the write sometimes reach a machine without the new deployment. */
81function edge(cutoverMs: number, version: (at: number, n: number) => string | null, status = 200): RouteIo & { calls: number } {
82 let now = 0
83 let n = 0
84 const io = {
85 calls: 0,
86 now: () => now,
87 sleep: async (ms: number) => {
88 now += ms
89 },
90 get: async () => {
91 n += 1
92 io.calls += 1
93 now += 1
94 return { status, version: version(now, n), ray: `ray${n}-${n % 2 === 0 ? 'IAD' : 'EWR'}` }
95 },
96 }
97 void cutoverMs
98 return io
99}
101const TUNABLES = { batch: 10, intervalMs: 1000, sustainMs: 10_000, maxMs: 120_000, requestMs: 2000 }
103test('replay 20-48-18: the old five-in-a-row check passed at +5 s, the straggler at +9.9 s; the sustained check confirms only after it', async () => {
104 // one old answer in a hundred until +9.9 s, exactly the shape the 2026-11-10 run saw
105 const io = edge(9_900, (at, n) => (at <= 9_900 && n % 37 === 0 ? OLD : NEW))
106 const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES)
107 assert.equal(verdict.kind, 'confirmed')
108 if (verdict.kind !== 'confirmed') return
109 const late = stragglers(verdict.samples, NEW)
110 assert.ok(late.length > 0)
111 const lastLate = Math.max(...late.map(item => item.at))
112 assert.ok(verdict.sustainedFrom > lastLate, `sustained window ${verdict.sustainedFrom} must start after the last straggler ${lastLate}`)
113 assert.ok(verdict.at - verdict.sustainedFrom >= TUNABLES.sustainMs)
114 assert.ok(verdict.samples.length >= 100)
115})
117test('a version that never settles is routing (harness), not a product failure', async () => {
118 const io = edge(Infinity, (_at, n) => (n % 50 === 0 ? OLD : NEW))
119 const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES)
120 assert.equal(verdict.kind, 'routing')
121 if (verdict.kind === 'routing') assert.match(verdict.reason, /^routing: /)
122})
124test('answers with no x-ply-version at all are the Worker', async () => {
125 const io = edge(0, () => null)
126 const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES)
127 assert.equal(verdict.kind, 'worker')
128})
130test('lost answers never confirm a route', async () => {
131 const io = edge(0, () => null, 0)
132 const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES)
133 assert.equal(verdict.kind, 'routing')
134})
136// ply-web#146: the repro from the issue (every GET answers HTTP 500 with the wanted x-ply-version). A served 5xx is the
137// product, never routing, and it is decided on the first such answer, not after maxMs.
138const PIN_HEADERS = { 'Cloudflare-Workers-Version-Overrides': `ply="${NEW}"` }
140test('#146 pin: every answer HTTP 500 from the wanted version is the Worker (served_5xx), at once', async () => {
141 const io = edge(0, () => NEW, 500)
142 const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: PIN_HEADERS, since: 0 }, io, TUNABLES)
143 assert.equal(verdict.kind, 'worker')
144 if (verdict.kind === 'worker') assert.equal(verdict.cause, 'served_5xx')
145 assert.equal(io.calls, TUNABLES.batch, 'decided after the first round, not after maxMs')
146 assert.equal(failureFault(verdict), 'worker')
147})
150test('#146 one intermittent 5xx from the wanted version among 2xx answers is the Worker', async () => {
151 let n = 0
152 let now = 0
153 const io: RouteIo = {
154 now: () => now,
155 sleep: async ms => {
156 now += ms
157 },
158 get: async () => {
159 n += 1
160 now += 1
161 return { status: n === 57 ? 503 : 200, version: NEW, ray: `r${n}-IAD` }
162 },
163 }
164 const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES)
165 assert.equal(verdict.kind, 'worker')
166 if (verdict.kind === 'worker') assert.equal(verdict.cause, 'served_5xx')
167})
169test('#146 a 5xx from another version is still routing, and other-version answers mixed with 2xx from the wanted version are routing', async () => {
170 const mixed = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, edge(Infinity, (_at, n) => (n % 50 === 0 ? OLD : NEW)), TUNABLES)
171 assert.equal(mixed.kind, 'routing')
172 assert.equal(failureFault(mixed), 'harness_fault')
173 const old5xx = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, edge(0, () => OLD, 500), TUNABLES)
174 assert.equal(old5xx.kind, 'routing')
175})
177test('#146 confirmed has no fault; headerless answers are the Worker (no_version_header)', async () => {
178 const ok = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, edge(0, () => NEW), TUNABLES)
179 assert.equal(ok.kind, 'confirmed')
180 const bare = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, edge(0, () => null), TUNABLES)
181 assert.equal(bare.kind, 'worker')
182 if (bare.kind === 'worker') assert.equal(bare.cause, 'no_version_header')
183})
185// ply-web#146 through the three callers that turn a verdict into a pipeline fault: the pin gate (stage-deploy pinReady),
186// the live check (pipeline liveRoute, judged by liveVerdict) and the smoke restart (stage-smoke reconfirmRoute). Each
187// fake records the headers it was sent, so the pin path is shown to carry the override and the live path to carry none.
188function recording(status: number, version: (n: number) => string | null): RouteIo & { readonly sent: Record<string, string>[] } {
189 let now = 0
190 let n = 0
191 const sent: Record<string, string>[] = []
192 return {
193 sent,
194 now: () => now,
195 sleep: async ms => {
196 now += ms
197 },
198 get: async (_url, headers) => {
199 n += 1
200 now += 1
201 sent.push(headers)
202 return { status, version: version(n), ray: `r${n}-IAD` }
203 },
204 }
205}
206const SITE = { baseUrl: 'https://x', worker: 'ply' }
207const OVERRIDE = { 'Cloudflare-Workers-Version-Overrides': `ply="${NEW}"` }
209test('#146 pin path: pinReady fails with fault worker when the canary serves HTTP 500 on the override', async () => {
210 const io = recording(500, () => NEW)
211 const result = await pinReady(SITE, NEW, 0, io, TUNABLES)
212 assert.deepEqual(io.sent[0], OVERRIDE)
213 assert.equal(result.ok, false)
214 if (result.ok) return
215 assert.equal(result.fault, 'worker')
216 assert.match(result.reason, /served HTTP 500/)
217 assert.equal(io.sent.length, TUNABLES.batch, 'decided after the first round')
218})
220test('#146 live path: liveRoute returns a Fail with fault worker, and liveVerdict rolls back', async () => {
221 const io = recording(500, () => NEW)
222 const lines: string[] = []
223 const result = await liveRoute(SITE, { version: NEW, log: line => lines.push(line) }, io, TUNABLES)
224 assert.deepEqual(io.sent[0], {})
225 assert.notEqual(result, null)
226 if (result === null) return
227 assert.equal(result.fault, 'worker')
228 assert.equal(result.stage, 'smoke')
229 assert.equal(liveVerdict([result]), 'rollback')
230 assert.match(lines.join('\n'), /route-confirm live: worker/)
231})
233test('#146 live path: other-version answers mixed with 2xx from the wanted version halt (harness_fault), never roll back', async () => {
234 const io = recording(200, n => (n % 50 === 0 ? OLD : NEW))
235 const result = await liveRoute(SITE, { version: NEW, log: () => {} }, io, TUNABLES)
236 assert.notEqual(result, null)
237 if (result === null) return
238 assert.equal(result.fault, 'harness_fault')
239 assert.equal(liveVerdict([result]), 'halt')
240})
242test('#146 live path: a confirmed route is no failure', async () => {
243 assert.equal(await liveRoute(SITE, { version: NEW, log: () => {} }, recording(200, () => NEW), TUNABLES), null)
244})
246test('#146 smoke restart: reconfirmRoute fails with fault worker on a served 5xx, pinned and unpinned', async () => {
247 const pinned = recording(503, () => NEW)
248 const pin = await reconfirmRoute(SITE, NEW, true, pinned, TUNABLES)
249 assert.deepEqual(pinned.sent[0], OVERRIDE)
250 assert.equal(pin.ok ? null : pin.fault, 'worker')
251 const unpinned = recording(503, () => NEW)
252 const live = await reconfirmRoute(SITE, NEW, false, unpinned, TUNABLES)
253 assert.deepEqual(unpinned.sent[0], {})
254 assert.equal(live.ok ? null : live.fault, 'worker')
255})
257// The rule is status >= 500 (the issue). A 4xx from the wanted version is not a served 5xx: it never confirms (sustainedFrom
258// needs 2xx/3xx), so it ends as routing at maxMs, not as an immediate Worker verdict.
259for (const status of [404, 499]) {
260 test(`#146 boundary: HTTP ${status} from the wanted version is not served_5xx`, async () => {
261 const io = edge(0, () => NEW, status)
262 const verdict = await confirmRoute({ url: 'https://x/', version: NEW, headers: {}, since: 0 }, io, TUNABLES)
263 assert.equal(verdict.kind, 'routing')
264 assert.ok(io.calls > TUNABLES.batch, 'not decided on the first round')
265 })
266}
268// ---- smoke row attribution ----
270function stalls(list: readonly Stall[]): StallSource {
271 const overlap = (stall: Stall, from: number, to: number) => Math.max(0, Math.min(stall.to, to) - Math.max(stall.from, from))
272 return {
273 check: () => null,
274 during: (from, to) => list.filter(stall => overlap(stall, from, to) > 0),
275 stalledMs: (from, to) => list.reduce((sum, stall) => sum + overlap(stall, from, to), 0),
276 }
277}
279test('a smoke row served by another version is a routing event, never a pass or a product failure', () => {
280 const event = harnessEvent({ started: 0, ended: 50, served: OLD, status: 200 }, NEW, NO_STALLS)
281 assert.equal(event && event.kind, 'wrong_version')
282 assert.equal(harnessEvent({ started: 0, ended: 50, served: NEW, status: 200 }, NEW, NO_STALLS), null)
283 // no header at all is the Worker's to answer for (judged by the row), not routing
284 assert.equal(harnessEvent({ started: 0, ended: 50, served: null, status: 200 }, NEW, NO_STALLS), null)
285 // the 2026-11-10 replay: flags-rules answered by 480225db
286 const replay = harnessEvent({ started: 9_800, ended: 9_900, served: OLD, status: 200 }, NEW, NO_STALLS)
287 assert.ok(replay && replay.kind === 'wrong_version' && replay.served === OLD)
288})
290test('a lost smoke row across a client stall is a client_stall event; without a stall it stays a loss', () => {
291 const source = stalls([{ from: 1_000, to: 54_733, ms: 53_733 }])
292 const event = harnessEvent({ started: 500, ended: 55_000, served: null, status: 0 }, NEW, source)
293 assert.ok(event && event.kind === 'client_stall' && event.stalledMs === 53_733)
294 assert.equal(harnessEvent({ started: 60_000, ended: 62_000, served: null, status: 0 }, NEW, source), null)
295})
297test('only GETs are re-issued; a write restarts the lifecycle', () => {
298 assert.equal(reissuable({ method: 'GET' }), true)
299 assert.equal(reissuable({ method: 'POST' }), false)
300 assert.equal(reissuable({ method: 'DELETE' }), false)
301})
303// ---- probe schedule extension ----
305function policy(): RolloutPolicy {
306 const raw = JSON.parse(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')) as Record<string, unknown>
307 raw.ingestPollMs = 0
308 const parsed = parsePolicy(JSON.stringify(raw)).policy
309 return { ...parsed, steps: [{ percent: 100, dwellMs: 40_000 }] }
310}
312function judge(): ProbeJudge {
313 const rollout = policy()
314 return {
315 maxLossRate: rollout.probe.maxLossRate,
316 respondedMin: rollout.burn.minSamples,
317 minSamples: rollout.probe.minSamples,
318 stallLatencyMs: rollout.probe.stallLatencyMs,
319 fivexxHard: rollout.gates.fivexx === 'hard',
320 }
321}
323function answered(id: string, version: string, latencyMs: number, status = 200): ProbeOutcome {
324 return classifyProbe(id, { kind: 'response', status, ray: some('ray'), latencyMs, version: some(version) })
325}
327/**
328 * A process that freezes for `pauseMs` at `pauseAt`: the clock jumps, the monitor (driven by the same clock) sees it,
329 * and the probes in flight across it never answer. Probes issued otherwise answer after `latencyMs`.
330 */
331function frozenBox(pauseAt: number, pauseMs: number, latencyMs: number) {
332 let at = 0
333 let paused = false
334 const seen: Stall[] = []
335 let last = 0
336 const health: StallSource = {
337 check: () => {
338 if (at - last > 1000) {
339 const stall = { from: last, to: at, ms: at - last }
340 seen.push(stall)
341 last = at
342 return stall
343 }
344 last = at
345 return null
346 },
347 during: (from, to) => seen.filter(stall => Math.min(stall.to, to) - Math.max(stall.from, from) > 0),
348 stalledMs: (from, to) => seen.reduce((sum, stall) => sum + Math.max(0, Math.min(stall.to, to) - Math.max(stall.from, from)), 0),
349 }
350 const waiting: { due: number; resolve: () => void }[] = []
351 const clock: ProbeClock = {
352 now: () => at,
353 sleep: async ms => {
354 // the monitor ticks every 100 ms while the process runs; a freeze is one jump it sees afterwards
355 let left = ms
356 while (left > 0) {
357 if (!paused && at + Math.min(left, 100) > pauseAt) {
358 paused = true
359 at += pauseMs
360 }
361 const step = Math.min(left, 100)
362 at += step
363 left -= step
364 health.check()
365 }
366 for (const item of waiting.splice(0)) {
367 if (item.due <= at) item.resolve()
368 else waiting.push(item)
369 }
370 await Promise.resolve()
371 },
372 }
373 const issue = (seq: number) => {
374 const issuedAt = at
375 const crosses = !paused && issuedAt + latencyMs > pauseAt
376 const promise = new Promise<ProbeOutcome>(resolve => {
377 // its client timeout fires as soon as the process runs again: a timed_out loss, unless the stall is seen
378 if (crosses) {
379 waiting.push({ due: issuedAt + 2000, resolve: () => resolve({ kind: 'lost', id: `p-${seq}`, reason: 'timed_out' }) })
380 return
381 }
382 waiting.push({ due: issuedAt + latencyMs, resolve: () => resolve(answered(`p-${seq}`, NEW, latencyMs)) })
383 })
384 return { id: `p-${seq}`, promise }
385 }
386 return { clock, health, issue, read: () => at, seen }
387}
389test('replay 20-54-06: a 53.7 s freeze mid-dwell extends the window to minSamples real samples and nothing is lost', async () => {
390 const rollout = policy()
391 const box = frozenBox(5_000, 53_733, 50)
392 const outcomes = await runProbeSchedule({
393 clock: box.clock,
394 dwellMs: 40_000,
395 periodMs: 1000 / rollout.probe.rps,
396 maxInFlight: rollout.probe.maxInFlight,
397 batch: 2,
398 graceMs: rollout.probe.timeoutMs,
399 health: box.health,
400 extension: { maxMs: 120_000, minReal: rollout.probe.minSamples },
401 issue: box.issue,
402 shed: seq => ({ kind: 'shed', id: `shed-${seq}` }),
403 stop: () => {},
404 })
405 const window = judgeProbeWindow(outcomes, judge())
406 assert.equal(box.seen.length, 1)
407 assert.ok(window.loss.stalled >= 1)
408 assert.equal(window.loss.count, 0, 'probes in flight across the freeze are stalled, not lost')
409 assert.equal(window.loss.shed, 0, 'missed ticks are skipped, not burst into sheds')
410 assert.ok(window.loss.total >= rollout.probe.minSamples, `${window.loss.total} real samples`)
411 assert.equal(window.kind, 'ok')
412 // the dwell ran its 40 s of healthy time plus the freeze, not more than the cap
413 assert.ok(box.read() >= 40_000 + 53_733 && box.read() <= 40_000 + 120_000 + rollout.probe.timeoutMs + 1000)
414})
416test('a freeze longer than the extension cap is harness_unhealthy, never a product verdict', async () => {
417 const rollout = policy()
418 const box = frozenBox(5_000, 400_000, 50)
419 const outcomes = await runProbeSchedule({
420 clock: box.clock,
421 dwellMs: 40_000,
422 periodMs: 1000 / rollout.probe.rps,
423 maxInFlight: rollout.probe.maxInFlight,
424 batch: 2,
425 graceMs: rollout.probe.timeoutMs,
426 health: box.health,
427 extension: { maxMs: 120_000, minReal: rollout.probe.minSamples },
428 issue: box.issue,
429 shed: seq => ({ kind: 'shed', id: `shed-${seq}` }),
430 stop: () => {},
431 })
432 const window = judgeProbeWindow(outcomes, judge())
433 assert.equal(window.kind, 'harness_unhealthy')
434 assert.match(window.kind === 'harness_unhealthy' ? window.reason : '', /^harness unhealthy: /)
435})
437test('without a stall the dwell is not extended', async () => {
438 const rollout = policy()
439 const box = frozenBox(Infinity, 0, 50)
440 await runProbeSchedule({
441 clock: box.clock,
442 dwellMs: 10_000,
443 periodMs: 1000 / rollout.probe.rps,
444 maxInFlight: rollout.probe.maxInFlight,
445 batch: 2,
446 graceMs: rollout.probe.timeoutMs,
447 health: box.health,
448 extension: { maxMs: 120_000, minReal: rollout.probe.minSamples },
449 issue: box.issue,
450 shed: seq => ({ kind: 'shed', id: `shed-${seq}` }),
451 stop: () => {},
452 })
453 assert.equal(box.seen.length, 0)
454 // the dwell plus at most the in-flight grace, never the 120 s extension
455 assert.ok(box.read() <= 10_000 + rollout.probe.timeoutMs + 100, String(box.read()))
456})
458test('judge: stalled probes leave the denominator; a served 5xx still rolls back', () => {
459 const outcomes: ProbeOutcome[] = []
460 for (let index = 0; index < 53; index += 1) outcomes.push(answered(`ok-${index}`, NEW, 50))
461 for (let index = 0; index < 5; index += 1) outcomes.push({ kind: 'stalled', id: `s-${index}` })
462 const unhealthy = judgeProbeWindow(outcomes, judge())
463 assert.equal(unhealthy.kind, 'harness_unhealthy')
464 assert.equal(unhealthy.loss.total, 53)
465 assert.equal(unhealthy.loss.count, 0)
466 const withFive = [...outcomes, answered('five', NEW, 50, 500)]
467 assert.equal(judgeProbeWindow(withFive, judge()).kind, 'rollback')
468 const enough: ProbeOutcome[] = []
469 for (let index = 0; index < judge().minSamples; index += 1) enough.push(answered(`ok-${index}`, NEW, 50))
470 enough.push({ kind: 'stalled', id: 'client-stall' })
471 assert.equal(judgeProbeWindow(enough, judge()).kind, 'ok')
472})
474function stats(requests: number): VersionStats {
475 return {
476 requests,
477 invocations: requests,
478 exceptions: 0,
479 errors: 0,
480 fivexx: 0,
481 errorLogs: 0,
482 wallMs: [10, 10, 10, 10, 10],
483 cpuMs: [10],
484 fetchMs: [10],
485 v1: [],
486 internal: [],
487 }
488}
490function unhealthyRun(rollout: RolloutPolicy) {
491 const counts = { rolled: 0, splits: [] as number[], dwells: 0 }
492 const outcomes: ProbeOutcome[] = []
493 for (let index = 0; index < 53; index += 1) outcomes.push(answered(`ok-${index}`, NEW, 50))
494 outcomes.push({ kind: 'stalled', id: 'client-stall 0-53733 53733ms' })
495 const run = runRamp(rollout, {
496 now: () => 0,
497 sleep: async () => {},
498 dwell: async () => {
499 counts.dwells += 1
500 return { outcomes }
501 },
502 applySplit: async (percent: number) => {
503 counts.splits.push(percent)
504 return null
505 },
506 observe: async (): Promise<Observation> => {
507 const next = stats(200)
508 return { kind: 'sample', next, previous: stats(0), aggregate: next, landed: 200 }
509 },
510 rollback: async () => {
511 counts.rolled += 1
512 return null
513 },
514 })
515 return { run, counts }
516}
518// ply-web#145: the step's split write already moved traffic, so an unjudged step must settle it like a harness fault.
519test('runRamp: an unhealthy client above 0% rolls back under the checked-in onHarnessFault policy and says so', async () => {
520 const rollout = { ...policy(), steps: [{ percent: 10, dwellMs: 40_000 }] }
521 assert.equal(rollout.probe.onHarnessFault, 'telemetry')
522 const { run, counts } = unhealthyRun(rollout)
523 const result = await run
524 assert.equal(counts.rolled, 1)
525 assert.deepEqual(counts.splits, [10])
526 assert.equal(counts.dwells, 1)
527 assert.equal(result.state.kind, 'held')
528 if (result.state.kind === 'held') {
529 assert.equal(result.state.outcome, 'harness_unhealthy')
530 assert.equal('traffic' in result.state ? result.state.traffic : null, 'rolled_back')
531 assert.doesNotMatch(result.state.reason, /prod untouched/)
532 assert.match(result.state.reason, /^harness unhealthy: /)
533 assert.match(result.state.reason, /rolled back to previous$/)
534 }
535})
537test('runRamp: an unhealthy client under onHarnessFault hold with no passed step rolls back', async () => {
538 const base = policy()
539 const rollout = { ...base, steps: [{ percent: 10, dwellMs: 40_000 }], probe: { ...base.probe, onHarnessFault: 'hold' as const } }
540 const { run, counts } = unhealthyRun(rollout)
541 const result = await run
542 assert.equal(counts.rolled, 1)
543 assert.deepEqual(counts.splits, [10])
544 assert.equal(result.state.kind === 'held' && 'traffic' in result.state ? result.state.traffic : null, 'rolled_back')
545})
Diagnostics4 git calls · 29 ms
CallStatusSizeTime029 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit7 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=4 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2006.6 KiB · hit20 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=20 git.response.bytes=6743 git.response.cache=HIT git.response.duration_ms=20 git.fetch.wants=478db28d705fdc4a444fa667d9714ebe6d1c23a0 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=6743 git.pkt.payload.max=6722 git.sideband.band1.bytes=6721 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e394be7aa26019
- Request
- afd93867…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format