main
1import { assertNever } from '../../src/never.ts'
2import { none, some, type Option } from '../../src/option.ts'
3import { compareStep, observedBasis, observeStep, predictStep, probeBasis, type ForecastBasis, type ForecastCheck } from './forecast.ts'
4import { asRecord } from './json.ts'
5import type { ObservationRef } from './log.ts'
6import type { RolloutPolicy } from './policy.ts'
7import { emptyProbeLoss, judgeProbeWindow, type ProbeJudge, type ProbeLoss, type ProbeOutcome, type ProbeWindow } from './probe.ts'
8import { percentile, type VersionStats } from './telemetry.ts'
9import { burnMap, CARRY_WALL_SAMPLES, judgeStep, routedShare, type RoutedShare, type StepVerdict } from './slo.ts'
11export type { VersionStats } from './telemetry.ts'
13/**
14 * Which invocations a step's latency figures (wall and CPU p99, the forecast's wall delta) are read from. Counts, errors,
15 * 5xx, burn and routed share always come from every invocation of the window; only latency is scoped.
16 *
17 * `probes`: the step's own responded probes, joined on cf-ray. The ramp's probes are the one traffic the harness controls
18 * at every step, so the figure means the same thing at 10% and at 100%. At 100% the live smoke, the visual smoke and the
19 * rate gate run inside the dwell (pipeline.ts `liveChecks`); their writes wait on Artifacts. Measured on run
20 * 2026-11-10T14-56-52-021Z (480225db, the 100% step's query window 15:04:07.9Z to 15:05:47.9Z, re-queried read-only on
21 * 2026-11-10): 2,091 external invocations of the new version, of which 1,962 probes (wall p99 14 ms, cpu p99 7), 74
22 * live-smoke requests (p99 6,580 ms, the slowest POST /v1/branch) and 55 GET /view from the visual smoke (p99 836 ms).
23 * The step reported p99 696 ms from all of them; its probes alone were 14 ms, beside 76/40/49/15 ms at 10..75%.
24 * `all`: no probe answered with a ray (no dwell, or every probe lost): every invocation, and the line says so.
25 */
26export type LatencySample =
27 | { readonly kind: 'probes'; readonly rays: readonly string[] }
28 | { readonly kind: 'all'; readonly reason: 'no_probe_rays' }
30export const ALL_INVOCATIONS: LatencySample = { kind: 'all', reason: 'no_probe_rays' }
32/** The responded probes' cf-rays, or every invocation when there are none. */
33export function latencySampleOf(outcomes: readonly ProbeOutcome[]): LatencySample {
34 const rays: string[] = []
35 for (const outcome of outcomes) {
36 if (outcome.kind === 'responded' && outcome.ray.tag === 'some') rays.push(outcome.ray.value)
37 }
38 return rays.length > 0 ? { kind: 'probes', rays } : ALL_INVOCATIONS
39}
41export type Observation =
42 | { readonly kind: 'unavailable'; readonly reason: string; readonly ref?: ObservationRef }
43 | {
44 readonly kind: 'sample'
45 readonly next: VersionStats
46 readonly previous: VersionStats
47 readonly aggregate: VersionStats
48 /**
49 * The window's invocations of the latency sample (LatencySample): with `probes`, the probes that have reached the
50 * telemetry; with `all`, every external invocation. The polled ingest wait counts these: at 100% the live checks'
51 * own requests are in `aggregate`, and counting them stopped the wait on 2,024 events of which only about 1,950
52 * were probes (run 2026-11-10T14-56-52-021Z; 2,091 had landed when re-queried).
53 */
54 readonly landed: number
55 readonly ref?: ObservationRef
56 }
58/** A query the ramp could not run. This is not a rollback: no observed burn or error was seen. */
59export type HarnessRamp = {
60 readonly outcome: 'harness_fault'
61 readonly reason: string
62 readonly rules: StepVerdict['rules']
63 readonly burns: StepVerdict['burns']
64}
66export type ProbeSampleDecision = {
67 readonly outcome: 'insufficient_samples'
68 readonly reason: string
69 readonly rules: StepVerdict['rules']
70 readonly burns: StepVerdict['burns']
71}
73/** The client itself stalled and the window could not be refilled: no verdict on the Worker; `runRamp` settles the step's traffic. */
74export type HarnessUnhealthyDecision = {
75 readonly outcome: 'harness_unhealthy'
76 readonly reason: string
77 readonly rules: StepVerdict['rules']
78 readonly burns: StepVerdict['burns']
79}
81export type RampDecision = StepVerdict | HarnessRamp | ProbeSampleDecision | HarnessUnhealthyDecision
83export type HeldWrite = { readonly kind: 'state_conflict'; readonly reason: string }
85declare const settledBy: unique symbol
87/**
88 * Where an unjudged step left traffic: rolled back to `previous`, held at the last fully passed split (or at 0%), or
89 * `settle_failed`: the rollback or hold write failed, so the new version is still at the stopped step's percentage
90 * (the reason names that percentage and the error; `rollbackError` carries the error). The brand is private to this module and only `settleHarnessFault` attaches it, so a state that stopped on an
91 * unjudged step cannot exist without its traffic having been settled (ply-web#145).
92 */
93export type HarnessTraffic = ('rolled_back' | 'held' | 'settle_failed') & { readonly [settledBy]: 'settleHarnessFault' }
95/** A step the ramp could not judge. Its split write already moved traffic, so `runRamp` must settle it. */
96export type UnjudgedOutcome = 'harness_fault' | 'insufficient_samples' | 'harness_unhealthy'
98/** Not a `RampState`: the step stopped and its traffic is not settled yet. Only `settleHarnessFault` consumes it. */
99export type Unsettled = { readonly kind: 'unsettled'; readonly reason: string; readonly outcome: UnjudgedOutcome }
101export type RampState =
102 | { readonly kind: 'ramping'; readonly step: number }
103 | { readonly kind: 'rolled-back'; readonly reason: string }
104 | { readonly kind: 'stopped'; readonly reason: string; readonly fault: 'state_conflict' }
105 | { readonly kind: 'stopped'; readonly reason: string; readonly fault: 'harness_fault'; readonly traffic: HarnessTraffic }
106 /** Judged on telemetry and held by policy (`split.onMismatch` / `onInsufficient`). */
107 | { readonly kind: 'held'; readonly reason: string; readonly outcome: 'hold' | 'insufficient_data' }
108 /** Unjudged (no product verdict); `traffic` says where `settleHarnessFault` left the split. */
109 | { readonly kind: 'held'; readonly reason: string; readonly outcome: 'insufficient_samples' | 'harness_unhealthy'; readonly traffic: HarnessTraffic }
110 | { readonly kind: 'complete' }
112export function rampStart(): RampState {
113 return { kind: 'ramping', step: 0 }
114}
116/**
117 * A query that never returned. Probes that already answered are the correctness signal:
118 * a 5xx is a rollback when that gate is hard, and a clean probe window advances. The
119 * missing series is named on the reason and is not treated as zero errors. With no
120 * probe sample there is nothing to judge, so the step stays a harness fault.
121 */
122export function decideWithoutTelemetry(probes: ProbeLoss | null, policy: RolloutPolicy, reason: string): RampDecision {
123 if (probes === null || probes.total === 0) {
124 return {
125 outcome: 'harness_fault',
126 reason,
127 rules: [{ name: 'telemetry_required', value: 1 }],
128 burns: burnMap([]),
129 }
130 }
131 if (probes.serverErrors > 0 && policy.gates.fivexx === 'hard') {
132 return {
133 outcome: 'rollback',
134 reason: `${probes.serverErrors} 5xx on responded probes; telemetry query lost; ${reason}`,
135 rules: [{ name: 'max_new_5xx', value: 0 }],
136 burns: burnMap([]),
137 }
138 }
139 return {
140 outcome: 'advance',
141 reason: `informational: telemetry query lost; ${probes.responded} responded probes, ${probes.serverErrors} server errors; ${reason}`,
142 rules: [
143 { name: 'telemetry.query.attempts', value: policy.telemetry.query.attempts },
144 { name: 'probe_responded', value: probes.responded },
145 ],
146 burns: burnMap([]),
147 }
148}
150/** One ramp step. A returned sample is judged as usual. A lost query uses the probe statuses. */
151export function decideRampStep(
152 obs: Observation,
153 policy: RolloutPolicy,
154 targetPercent: number,
155 probes: ProbeLoss | null = null,
156): RampDecision {
157 if (obs.kind === 'unavailable') return decideWithoutTelemetry(probes, policy, obs.reason)
158 return judgeStep({
159 next: obs.next,
160 previous: obs.previous,
161 aggregate: obs.aggregate,
162 targetPercent,
163 policy,
164 })
165}
167/** Advance on a clean step. A terminal state stays terminal. An unjudged step is `unsettled` until `runRamp` settles its traffic. */
168export function applyRamp(state: RampState, decision: RampDecision, stepCount: number): RampState | Unsettled {
169 if (state.kind !== 'ramping') return state
170 switch (decision.outcome) {
171 case 'harness_fault':
172 case 'insufficient_samples':
173 case 'harness_unhealthy':
174 return { kind: 'unsettled', reason: decision.reason, outcome: decision.outcome }
175 case 'rollback':
176 return { kind: 'rolled-back', reason: decision.reason }
177 case 'hold':
178 case 'insufficient_data':
179 return { kind: 'held', reason: decision.reason, outcome: decision.outcome }
180 case 'advance': {
181 const next = state.step + 1
182 if (next >= stepCount) return { kind: 'complete' }
183 return { kind: 'ramping', step: next }
184 }
185 default:
186 return assertNever(decision)
187 }
188}
190function fixedRate(count: number, total: number): string {
191 if (total <= 0) return '-'
192 return (count / total).toFixed(4)
193}
195function p99(values: readonly number[] | undefined): string {
196 if (!values) return '-'
197 const value = percentile(values, 0.99)
198 return value === null ? '-' : String(Math.round(value))
199}
201function points(fraction: number | null): string {
202 if (fraction === null) return '-'
203 return (fraction * 100).toFixed(2)
204}
206function tolerance(share: RoutedShare): string {
207 if (share.low === null || share.high === null) return '-'
208 return `${points(share.low)}..${points(share.high)}`
209}
211function entrypointList(rows: VersionStats['internal'] | undefined): string {
212 if (!rows || rows.length === 0) return ''
213 return rows.map(row => `${row.entrypoint}:${row.count}`).join(',')
214}
216/** Named-entrypoint fetches, display only. Empty when this window had none. */
217function internalNote(next: VersionStats | null, previous: VersionStats | null): string {
218 const newer = entrypointList(next?.internal)
219 const older = entrypointList(previous?.internal)
220 const nextField = newer === '' ? '' : ` internal_next=${newer}`
221 const previousField = older === '' ? '' : ` internal_previous=${older}`
222 return `${nextField}${previousField}`
223}
225/** One line an operator can paste. `step` is 1-based. Sample counts are external fetches in this window. Carried p99 is display only. */
226export function progressLine(input: {
227 readonly step: number
228 readonly steps: number
229 readonly percent: number
230 readonly next: VersionStats | null
231 readonly previous: VersionStats | null
232 readonly carriedPrevious: VersionStats | null
233 readonly share: RoutedShare
234 readonly decision: RampDecision
235 readonly latency: RolloutPolicy['gates']['latency']['mode']
236 readonly probe: { readonly kind: 'none' } | { readonly kind: 'measured'; readonly loss: ProbeLoss }
237 readonly sample: LatencySample
238}): string {
239 const previousPercent = input.percent >= 100 ? 0 : 100 - input.percent
240 const samples = `${input.next?.requests ?? '-'}/${input.previous?.requests ?? '-'}`
241 const errors = `${input.next ? fixedRate(input.next.errors, input.next.requests) : '-'}/${input.previous ? fixedRate(input.previous.errors, input.previous.requests) : '-'}`
242 const oldWalls = input.carriedPrevious?.wallMs ?? input.previous?.wallMs
243 const oldCpu = input.carriedPrevious?.cpuMs ?? input.previous?.cpuMs
244 const walls = `${p99(input.next?.wallMs)}/${p99(oldWalls)}`
245 const cpu = `${p99(input.next?.cpuMs)}/${p99(oldCpu)}`
246 const latency = input.latency === 'off' ? '' : ` p99_wall_ms=${walls} p99_cpu_ms=${cpu} latency=${input.latency} latency_sample=${input.sample.kind}`
247 const routed = `routed_share=${points(input.share.share)} target=${input.percent} tolerance=${tolerance(input.share)} agg=${input.share.n}`
248 const internal = internalNote(input.next, input.previous)
249 const probe = input.probe.kind === 'measured'
250 ? ` probe_loss=${input.probe.loss.count}/${input.probe.loss.total} timed_out=${input.probe.loss.timedOut} shed=${input.probe.loss.shed}`
251 : ''
252 const head = `ramp step=${input.step}/${input.steps} percent=${input.percent} previous=${previousPercent} samples=${samples} error_rate=${errors} ${routed}${internal}${latency}${probe} decision=${input.decision.outcome}`
253 const carried = input.carriedPrevious ? ' previous_sample=carried' : ''
254 const informational = input.decision.outcome === 'advance' && input.decision.reason.includes('informational:')
255 if (input.decision.outcome === 'advance' && !informational) return head + carried
256 const reason = input.decision.reason.replace(/[\n\r"]/g, "'")
257 return `${head}${carried} reason="${reason}"`
258}
260export function readPassVerdict(text: string, version: string): { readonly ok: true } | { readonly ok: false; readonly reason: string } {
261 let value: unknown
262 try {
263 value = JSON.parse(text) as unknown
264 } catch {
265 return { ok: false, reason: 'verdict is not JSON' }
266 }
267 const rec = asRecord(value)
268 if (!rec) return { ok: false, reason: 'verdict is not an object' }
269 if (rec.version !== version) return { ok: false, reason: `verdict is for ${String(rec.version)}, not ${version}` }
270 if (Array.isArray(rec.failures) && rec.failures.length > 0) return { ok: false, reason: 'verdict has failures' }
271 if (rec.kind === 'pass' || rec.pass === true) return { ok: true }
272 return { ok: false, reason: 'verdict is not a pass' }
273}
275export type RampIo = {
276 now(): number
277 sleep(ms: number): Promise<void>
278 /** Probe during the dwell. When omitted, `sleep` is the whole dwell and no probe sample is judged. */
279 dwell?(ms: number, percent: number): Promise<{ readonly outcomes: readonly ProbeOutcome[] }>
280 applySplit(percent: number): Promise<null | string | HeldWrite>
281 /** `sample` scopes the latency figures only (LatencySample); every count and gate reads the whole window. */
282 observe(from: number, to: number, percent: number, sample: LatencySample): Promise<Observation>
283 /** Called once a step's split is written (or already in place). The pipeline starts the live checks at 100% here. */
284 onApplied?(percent: number): void
285 /** Called with the observation ref. A missing ref is a bug: a decision without an observation. */
286 onDecision?(ref: ObservationRef, decision: RampDecision, percent: number): Promise<void>
287 rollback(): Promise<null | string | HeldWrite>
288}
290export type RampStepRecord = {
291 readonly percent: number
292 readonly burns: RampDecision['burns']
293 readonly outcome: RampDecision['outcome']
294 readonly line: string
295 readonly probeLoss: ProbeLoss
296 /** What the step predicted next to what it saw (scripts/cd/forecast.ts). Null when the step had no telemetry sample. */
297 readonly forecast: ForecastCheck | null
298}
300export type RampLoopResult = {
301 readonly state: RampState
302 readonly lines: readonly string[]
303 readonly rollbackError: string | null
304 readonly steps: readonly RampStepRecord[]
305}
307/**
308 * Step the policy. At 100% the previous version is no longer allocated, so the old
309 * wall and CPU numbers are the last step whose old version had enough wall samples.
310 */
311function asConflict(value: null | string | HeldWrite): HeldWrite | null {
312 if (value && typeof value === 'object') return value
313 return null
314}
316const UNTESTED_SHARE: RoutedShare = { n: 0, newRequests: 0, share: null, low: null, high: null, kind: 'untested' }
318type Settled = { readonly state: RampState; readonly rollbackError: string | null }
320const ROLLED_BACK = 'rolled_back' as HarnessTraffic
321const HELD = 'held' as HarnessTraffic
322const SETTLE_FAILED = 'settle_failed' as HarnessTraffic
324function settledState(unsettled: Unsettled, reason: string, traffic: HarnessTraffic): RampState {
325 switch (unsettled.outcome) {
326 case 'harness_fault':
327 return { kind: 'stopped', reason, fault: 'harness_fault', traffic }
328 case 'insufficient_samples':
329 case 'harness_unhealthy':
330 return { kind: 'held', reason, outcome: unsettled.outcome, traffic }
331 default:
332 return assertNever(unsettled.outcome)
333 }
334}
336async function rollbackTraffic(io: RampIo, percent: number, unsettled: Unsettled): Promise<Settled> {
337 const rolled = await io.rollback()
338 const conflict = asConflict(rolled)
339 if (conflict) return { state: { kind: 'stopped', reason: conflict.reason, fault: 'state_conflict' }, rollbackError: null }
340 if (typeof rolled === 'string') {
341 return {
342 state: settledState(unsettled, `${unsettled.reason}; rollback to previous failed: ${rolled}; still at ${percent}%`, SETTLE_FAILED),
343 rollbackError: rolled,
344 }
345 }
346 return { state: settledState(unsettled, `${unsettled.reason}; rolled back to previous`, ROLLED_BACK), rollbackError: null }
347}
349/**
350 * An unjudged step (harness fault, too few samples, unhealthy client). Its split write already moved traffic, so
351 * above 0% it rolls back to `previous` (`rollback`, `telemetry`), or holds the last fully passed split (`hold`; a
352 * rollback when no step passed). The returned state's reason says where traffic is, including when that write failed.
353 */
354async function settleHarnessFault(
355 policy: RolloutPolicy,
356 io: RampIo,
357 percent: number,
358 unsettled: Unsettled,
359 lastPassed: Option<number>,
360): Promise<Settled> {
361 if (percent <= 0) return { state: settledState(unsettled, `${unsettled.reason}; held at 0%`, HELD), rollbackError: null }
362 switch (policy.probe.onHarnessFault) {
363 case 'rollback':
364 // A telemetry-unavailable fault under `telemetry` has nothing left to judge on: restore the previous version.
365 case 'telemetry':
366 return rollbackTraffic(io, percent, unsettled)
367 case 'hold': {
368 if (lastPassed.tag === 'none' || lastPassed.value <= 0) return rollbackTraffic(io, percent, unsettled)
369 const held = await io.applySplit(lastPassed.value)
370 const conflict = asConflict(held)
371 if (conflict) return { state: { kind: 'stopped', reason: conflict.reason, fault: 'state_conflict' }, rollbackError: null }
372 if (typeof held === 'string') {
373 return {
374 state: settledState(unsettled, `${unsettled.reason}; hold at ${lastPassed.value}% failed: ${held}; still at ${percent}%`, SETTLE_FAILED),
375 rollbackError: held,
376 }
377 }
378 return { state: settledState(unsettled, `${unsettled.reason}; held at ${lastPassed.value}%`, HELD), rollbackError: null }
379 }
380 default:
381 return assertNever(policy.probe.onHarnessFault)
382 }
383}
385function withProbeRules(decision: RampDecision, loss: ProbeLoss): RampDecision {
386 return {
387 ...decision,
388 rules: [
389 ...decision.rules,
390 { name: 'probe_loss_count', value: loss.count },
391 { name: 'probe_loss_rate', value: loss.rate },
392 { name: 'probe_timed_out', value: loss.timedOut },
393 { name: 'probe_shed', value: loss.shed },
394 ],
395 }
396}
398function probeJudge(policy: RolloutPolicy): ProbeJudge {
399 return {
400 maxLossRate: policy.probe.maxLossRate,
401 respondedMin: policy.burn.minSamples,
402 minSamples: policy.probe.minSamples,
403 stallLatencyMs: policy.probe.stallLatencyMs,
404 fivexxHard: policy.gates.fivexx === 'hard',
405 }
406}
408function probeWindowDecision(policy: RolloutPolicy, window: ProbeWindow): RampDecision {
409 switch (window.kind) {
410 case 'harness_fault':
411 return {
412 outcome: 'harness_fault',
413 reason: window.reason,
414 rules: [{ name: 'probe_max_loss_rate', value: policy.probe.maxLossRate }],
415 burns: burnMap([]),
416 }
417 case 'insufficient_samples':
418 return {
419 outcome: 'insufficient_samples',
420 reason: window.reason,
421 rules: [{ name: 'probe_min_samples', value: policy.probe.minSamples }],
422 burns: burnMap([]),
423 }
424 case 'rollback':
425 return {
426 outcome: 'rollback',
427 reason: window.reason,
428 rules: [{ name: 'max_new_5xx', value: 0 }],
429 burns: burnMap([]),
430 }
431 case 'retry':
432 return {
433 outcome: 'insufficient_samples',
434 reason: window.reason,
435 rules: [{ name: 'probe_window_attempts', value: policy.probe.windowAttempts }],
436 burns: burnMap([]),
437 }
438 case 'harness_unhealthy':
439 return {
440 outcome: 'harness_unhealthy',
441 reason: window.reason,
442 rules: [{ name: 'probe_min_samples', value: policy.probe.minSamples }],
443 burns: burnMap([]),
444 }
445 case 'ok':
446 throw new Error('an ok probe window is not a fault')
447 default:
448 return assertNever(window)
449 }
450}
452/** A stalled dwell is re-run up to `windowAttempts`. The last retry becomes a hold, not a rollback. */
453async function probeWindows(
454 policy: RolloutPolicy,
455 io: RampIo,
456 dwell: (ms: number, percent: number) => Promise<{ readonly outcomes: readonly ProbeOutcome[] }>,
457 dwellMs: number,
458 percent: number,
459): Promise<{ readonly window: ProbeWindow; readonly sample: LatencySample }> {
460 const judge = probeJudge(policy)
461 let window: ProbeWindow = { kind: 'ok', loss: emptyProbeLoss(policy.probe.maxLossRate) }
462 let retryReason = 'harness/network stall'
463 const limit = policy.probe.windowAttempts
464 // every attempt's probes are in the step's observed window (it opens before the first), so all of them are the sample
465 const outcomes: ProbeOutcome[] = []
466 let attempt = 0
467 while (attempt < limit) {
468 attempt += 1
469 const probed = await dwell(dwellMs, percent)
470 outcomes.push(...probed.outcomes)
471 window = judgeProbeWindow(probed.outcomes, judge)
472 if (window.kind !== 'retry') return { window, sample: latencySampleOf(outcomes) }
473 retryReason = window.reason
474 if (attempt < limit && policy.probe.windowBackoffMs > 0) await io.sleep(policy.probe.windowBackoffMs)
475 }
476 return {
477 window: {
478 kind: 'insufficient_samples',
479 loss: window.loss,
480 reason: `${retryReason}; ${limit} windows exhausted; ${window.loss.serverErrors} server errors, no 5xx verdict`,
481 },
482 sample: latencySampleOf(outcomes),
483 }
484}
486/**
487 * Wait for the dwell's tail to be ingested, then observe. With `ingestPollMs` the window is
488 * re-queried every period until the worker has logged as many invocations as probes answered
489 * (they are the bulk of the window) or `ingestWaitMs` has passed; without it, one fixed sleep.
490 * Without probes there is no count to wait for, so the full wait runs.
491 */
492export async function observeAfterIngest(
493 policy: RolloutPolicy,
494 io: Pick<RampIo, 'now' | 'sleep' | 'observe'>,
495 from: number,
496 to: number,
497 percent: number,
498 probes: ProbeLoss | null,
499 sample: LatencySample,
500): Promise<Observation> {
501 if (policy.ingestPollMs <= 0 || probes === null || probes.responded === 0) {
502 await io.sleep(policy.ingestWaitMs)
503 return io.observe(from, to, percent, sample)
504 }
505 const deadline = io.now() + policy.ingestWaitMs
506 for (;;) {
507 await io.sleep(Math.min(policy.ingestPollMs, Math.max(0, deadline - io.now())))
508 const observed = await io.observe(from, to, percent, sample)
509 if (io.now() >= deadline) return observed
510 if (observed.kind === 'sample' && observed.landed >= probes.responded * INGEST_LANDED) return observed
511 }
512}
514/** Share of answered probes the worker must have logged before the polled ingest wait stops. Workers Observability drops a few. */
515export const INGEST_LANDED = 0.98
517export async function runRamp(policy: RolloutPolicy, io: RampIo): Promise<RampLoopResult> {
518 const lines: string[] = []
519 const steps: RampStepRecord[] = []
520 let state = rampStart()
521 let carried: VersionStats | null = null
522 let rollbackError: string | null = null
523 let lastPassed: Option<number> = none()
524 // The first step is predicted from the probe rate alone; each later one from the step before it.
525 let basis: ForecastBasis | null = io.dwell ? probeBasis(policy) : null
526 const cap = policy.steps.length
527 for (let guard = 0; guard < cap && state.kind === 'ramping'; guard++) {
528 const index = state.step
529 const step = policy.steps[index]
530 if (!step) {
531 state = { kind: 'rolled-back', reason: `schedule has no step ${index}` }
532 break
533 }
534 const percent = step.percent
535 const splitError = await io.applySplit(percent)
536 const splitConflict = asConflict(splitError)
537 if (splitConflict) {
538 state = { kind: 'stopped', reason: splitConflict.reason, fault: 'state_conflict' }
539 break
540 }
541 if (typeof splitError === 'string') {
542 const rolled = await io.rollback()
543 const rolledConflict = asConflict(rolled)
544 if (rolledConflict) {
545 state = { kind: 'stopped', reason: rolledConflict.reason, fault: 'state_conflict' }
546 break
547 }
548 rollbackError = typeof rolled === 'string' ? rolled : null
549 state = { kind: 'rolled-back', reason: splitError }
550 break
551 }
552 if (io.onApplied) io.onApplied(percent)
553 if (policy.warmupMs > 0) await io.sleep(policy.warmupMs)
554 const from = io.now()
555 let probeLoss = emptyProbeLoss(policy.probe.maxLossRate)
556 let sample: LatencySample = ALL_INVOCATIONS
557 const dwell = io.dwell
558 if (dwell) {
559 const probed = await probeWindows(policy, io, dwell, step.dwellMs, percent)
560 const window = probed.window
561 sample = probed.sample
562 probeLoss = window.loss
563 // `telemetry`: a probe-loss fault is the harness (client aborts, probe tail). Record it and judge the step on the
564 // new version's telemetry below; a 5xx the Worker served is in that telemetry.
565 const deferred = window.kind === 'harness_fault' && policy.probe.onHarnessFault === 'telemetry'
566 if (window.kind !== 'ok' && !deferred) {
567 const decision = probeWindowDecision(policy, window)
568 const noted = withProbeRules(decision, probeLoss)
569 const line = progressLine({
570 step: index + 1,
571 steps: policy.steps.length,
572 percent,
573 next: null,
574 previous: null,
575 carriedPrevious: null,
576 share: UNTESTED_SHARE,
577 decision: noted,
578 latency: policy.gates.latency.mode,
579 probe: { kind: 'measured', loss: probeLoss },
580 sample,
581 })
582 lines.push(line)
583 steps.push({ percent, burns: noted.burns, outcome: noted.outcome, line, probeLoss, forecast: null })
584 if (window.kind === 'harness_fault' || window.kind === 'harness_unhealthy' || window.kind === 'insufficient_samples' || window.kind === 'retry') {
585 // Unjudged: the split write above already moved traffic, so settle it (ply-web#145).
586 const outcome = window.kind === 'retry' ? 'insufficient_samples' : window.kind
587 const settled = await settleHarnessFault(policy, io, percent, { kind: 'unsettled', reason: window.reason, outcome }, lastPassed)
588 state = settled.state
589 rollbackError = settled.rollbackError
590 } else {
591 const rolled = await io.rollback()
592 const rolledConflict = asConflict(rolled)
593 if (rolledConflict) state = { kind: 'stopped', reason: rolledConflict.reason, fault: 'state_conflict' }
594 else {
595 rollbackError = typeof rolled === 'string' ? rolled : null
596 state = { kind: 'rolled-back', reason: window.reason }
597 }
598 }
599 break
600 }
601 } else {
602 await io.sleep(step.dwellMs)
603 }
604 const to = io.now()
605 // Wait for the dwell's own tail to land, then query that window. The next step
606 // has not started. The checked-in wait is above the 2026-11-01 gaps: 12640ms
607 // at 1% and 17313ms at 5% on run 2026-11-01T00-10-02-620Z.
608 const observed = await observeAfterIngest(policy, io, from, to, percent, io.dwell ? probeLoss : null, sample)
609 if (observed.kind === 'sample' && observed.previous.wallMs.length >= CARRY_WALL_SAMPLES) carried = observed.previous
610 const carriedPrevious = observed.kind === 'sample'
611 && percent === 100
612 && observed.previous.wallMs.length < CARRY_WALL_SAMPLES
613 && carried
614 && carried.wallMs.length >= CARRY_WALL_SAMPLES
615 ? carried
616 : null
617 const probes = io.dwell ? probeLoss : null
618 const decision = withProbeRules(decideRampStep(observed, policy, percent, probes), probeLoss)
619 const liveShare = observed.kind === 'sample'
620 ? routedShare(observed.next.invocations, observed.aggregate.invocations, percent, policy.split.z)
621 : UNTESTED_SHARE
622 const line = progressLine({
623 step: index + 1,
624 steps: policy.steps.length,
625 percent,
626 next: observed.kind === 'sample' ? observed.next : null,
627 previous: observed.kind === 'sample' ? observed.previous : null,
628 carriedPrevious,
629 share: liveShare,
630 decision,
631 latency: policy.gates.latency.mode,
632 probe: io.dwell ? { kind: 'measured', loss: probeLoss } : { kind: 'none' },
633 sample,
634 })
635 lines.push(line)
636 let forecast: ForecastCheck | null = null
637 if (observed.kind === 'sample') {
638 const seen = observeStep(observed, policy.split.z)
639 const predicted = basis === null ? null : predictStep(basis, percent, step.dwellMs, policy.split.z)
640 forecast = compareStep({ step: index + 1, percent, basis, predicted, observed: seen.resolution, wallDeltaMs: seen.wallDeltaMs })
641 basis = observedBasis(index + 1, percent, observed, step.dwellMs)
642 }
643 steps.push({ percent, burns: decision.burns, outcome: decision.outcome, line, probeLoss, forecast })
644 if (io.onDecision) {
645 if (!observed.ref) throw new Error('decision without an observation')
646 await io.onDecision(observed.ref, decision, percent)
647 }
648 const applied = applyRamp(state, decision, policy.steps.length)
649 if (applied.kind === 'unsettled') {
650 const settled = await settleHarnessFault(policy, io, percent, applied, lastPassed)
651 state = settled.state
652 rollbackError = settled.rollbackError
653 break
654 }
655 state = applied
656 if (state.kind === 'ramping' || state.kind === 'complete') lastPassed = some(percent)
657 if (state.kind === 'rolled-back') {
658 const rolled = await io.rollback()
659 const rolledConflict = asConflict(rolled)
660 if (rolledConflict) {
661 state = { kind: 'stopped', reason: rolledConflict.reason, fault: 'state_conflict' }
662 break
663 }
664 rollbackError = typeof rolled === 'string' ? rolled : null
665 break
666 }
667 }
668 if (state.kind === 'ramping') {
669 const rolled = await io.rollback()
670 const rolledConflict = asConflict(rolled)
671 if (rolledConflict) {
672 state = { kind: 'stopped', reason: rolledConflict.reason, fault: 'state_conflict' }
673 } else {
674 rollbackError = typeof rolled === 'string' ? rolled : null
675 state = { kind: 'rolled-back', reason: 'ramp did not finish' }
676 }
677 }
678 return { state, lines, rollbackError, steps }
679}
Diagnostics4 git calls · 32 ms
CallStatusSizeTime032 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2008.3 KiB · hit26 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=26 git.response.bytes=8497 git.response.cache=HIT git.response.duration_ms=26 git.fetch.wants=ab2f39cb2a424381ca79fa0e3270ae96fe844b78 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=8497 git.pkt.payload.max=8476 git.sideband.band1.bytes=8475 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2db693a7fc125
- Request
- 01d19051…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format