main
1import { createHash } from 'node:crypto'
2import { readFileSync } from 'node:fs'
3import { fileURLToPath } from 'node:url'
4import type { Json } from './log.ts'
6/** Checked-in rollout policy. Override a run with `--policy <path>`. */
7// Every numeric threshold in the policy file has a provenance entry in scripts/calibrate/thresholds.json: a cited
8// distribution or `PRIOR (uncalibrated)` (issue #73, test/calibrate-provenance.test.ts).
9export const DEFAULT_POLICY_PATH = fileURLToPath(new URL('./cd.policy.json', import.meta.url))
11export type SignalMode = 'hard' | 'informational'
13export type ProbeCall = {
14 readonly method: 'GET' | 'POST'
15 readonly path: string
16 readonly auth: boolean
17}
19/**
20 * Where a harness fault above 0% leaves traffic. Rollback restores `previous` at 100%.
21 * `telemetry` does not decide on the probe window at all: the loss is recorded, and the
22 * step is judged on the new version's own telemetry (a probe-tail loss is the harness,
23 * not the Worker). The checked-in ramp uses it since 2026-11-08: under `rollback`, runs
24 * 2026-11-06T17-32-11-293Z (6/2000) and 18-44-58-475Z (5/2000) rolled back clean versions on
25 * probe-tail loss alone.
26 */
27export type ProbeFaultAction = 'rollback' | 'hold' | 'telemetry'
29/**
30 * Citation for `probe.maxLossRate`, per issue #73: source, window, date, n, and headroom.
31 * The number is the 100% step of run 2026-11-01T17-06-08-780Z: 1 client abort in 790 probes.
32 */
33export type ProbeCalibration = {
34 readonly source: string
35 readonly date: string
36 readonly window: string
37 readonly n: number
38 readonly lost: number
39 readonly web: string
40 readonly ply: string
41 readonly observedRate: string
42 readonly percentile: string
43 readonly headroom: string
44}
46/**
47 * Citation for the probe scheduler, per issue #73.
48 * Run 2026-11-01T19-40-35-485Z (web 22bfb79), 50% step: healthy p50 41–101ms on
49 * both versions, then a client stall at ~69830ms. 128 issued, 10 never answered.
50 */
51export type ProbeScheduleCalibration = {
52 readonly source: string
53 readonly date: string
54 readonly window: string
55 readonly n: number
56 readonly responded: number
57 readonly lost: number
58 readonly healthyN: number
59 readonly web: string
60 readonly oldVersion: string
61 readonly newVersion: string
62 readonly healthyP50Ms: string
63 readonly observedStallMs: number
64 readonly dwellMs: number
65 readonly observedRate: string
66 readonly percentile: string
67 readonly headroom: string
68}
70export type RolloutStep = {
71 readonly percent: number
72 readonly dwellMs: number
73}
75/** `gate` without thresholds is not a value of this type. The parser rejects it. */
76export type LatencyPolicy =
77 | { readonly mode: 'off' }
78 | { readonly mode: 'informational' }
79 | { readonly mode: 'gate'; readonly minSamples: number; readonly p99Ratio: number; readonly effectFloorMs: number }
81export type RolloutGates = {
82 readonly smoke: SignalMode
83 readonly fivexx: SignalMode
84 readonly exceptions: SignalMode
85 readonly errorLogs: SignalMode
86 readonly nonOk: SignalMode
87 readonly burn: SignalMode
88 readonly latency: LatencyPolicy
89}
91/** What a routed-share miss does. It does not advance. */
92export type SplitAction = 'hold' | 'rollback'
94/** What too few new-version requests does. Rollback is not a choice: thin evidence is not a fault. */
95export type EvidenceAction = 'advance' | 'hold'
97/**
98 * Citation for scratch-repo delete, per issue #73.
99 * `latencyMs` is the timed-out delete on run 2026-11-01T17-29-43-122Z (web 9558b10): 103s, then 404.
100 */
101export type CleanupCalibration = {
102 readonly source: string
103 readonly date: string
104 readonly window: string
105 readonly n: number
106 readonly latencyMs: number
107 readonly web: string
108 readonly namespace: string
109 readonly observed: string
110 readonly percentile: string
111 readonly headroom: string
112}
114/** How a scratch repo is deleted. `attempts` bounds the retries. A miss is a leak, not a failed release. */
115export type CleanupPolicy = {
116 readonly attempts: number
117 readonly backoffMs: number
118 readonly staleMinutes: number
119 readonly calibration: CleanupCalibration
120}
122export type SplitPolicy = {
123 /** Half-width of the acceptance band, in standard errors of a binomial at the target percent. */
124 readonly z: number
125 /** New-version fetch invocations required before a step may advance. */
126 readonly minNewRequests: number
127 readonly onMismatch: SplitAction
128 readonly onInsufficient: EvidenceAction
129}
131/**
132 * Workers Observability drops whole invocations in bursts. A smoke id with no log line or no trace is
133 * telemetry loss: recorded with a count, rate and id list, and only fails the run above `maxLossRate`.
134 *
135 * `query` bounds retries of `cf o11y telemetry query`. The citation is run 2026-11-01T19-17-18-170Z:
136 * one timeout at the 100% step. A recovered query is a normal sample. A query that never returns
137 * does not invent zeroes for the series it did not fetch.
138 */
139export type QueryCalibration = {
140 readonly source: string
141 readonly date: string
142 readonly window: string
143 readonly n: number
144 readonly web: string
145 readonly ply: string
146 readonly observed: string
147 readonly percentile: string
148 readonly headroom: string
149}
151export type TelemetryQueryPolicy = {
152 readonly attempts: number
153 readonly backoffMs: number
154 readonly calibration: QueryCalibration
155}
157export type TelemetryPolicy = {
158 /** Fraction of smoke ids in (0, 1] that may be missing from telemetry before the run fails. */
159 readonly maxLossRate: number
160 readonly query: TelemetryQueryPolicy
161}
163/** Default when the policy omits `telemetry`: fail only when more than half of the smoke is missing. */
164export const DEFAULT_MAX_LOSS_RATE = 0.5
166/**
167 * The rate limiter's silent-degradation gate (scripts/cd/rate-gate.ts, docs/rate-limit.md "CD gate"). Every number has
168 * its evidence in `calibration`. Omitted from a policy file: DEFAULT_RATE_GATE (the same numbers as cd.policy.json).
169 */
170export type RateGatePolicy = {
171 /** The limiter every real request must run (RATE_MODE). A deliberate RATE_MODE rollback edits this too. */
172 readonly expectMode: 'batched' | 'memo' | 'legacy'
173 /** Pinned viewer requests the gate sends (load token, no mode header) so the version has default-mode checks. */
174 readonly probes: number
175 readonly probeGapMs: number
176 readonly probePath: string
177 /** Warm default-mode checks needed before the fail_open share is judged. Fewer after the wait is a harness fault. */
178 readonly minChecks: number
179 /** Warm (not the isolate's first) default-mode fail_open share above this fails the version. */
180 readonly maxFailOpenRate: number
181 /** Flushes dropped (deadline, overflow) without an injected fault, above this fails the version. */
182 readonly maxDroppedFlushes: number
183 readonly calibration: RateCalibration
184}
186export type RateCalibration = {
187 readonly source: string
188 readonly date: string
189 readonly n: string
190 readonly observed: string
191 readonly headroom: string
192}
194export const DEFAULT_RATE_GATE: RateGatePolicy = {
195 expectMode: 'memo',
196 probes: 30,
197 probeGapMs: 300,
198 probePath: '/view?remote=https%3A%2F%2F5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net%2Fgit%2Fpublic-v0%2Fply.git',
199 minChecks: 20,
200 maxFailOpenRate: 0.1,
201 maxDroppedFlushes: 0,
202 calibration: {
203 source: 'default for an omitted rate block; see cd.policy.json rate.calibration',
204 date: '2026-11-07',
205 n: 'see cd.policy.json',
206 observed: 'see cd.policy.json',
207 headroom: 'see cd.policy.json',
208 },
209}
211export type RolloutPolicy = {
212 readonly steps: readonly RolloutStep[]
213 readonly ingestWaitMs: number
214 /** When > 0, the post-dwell ingest wait is polled at this period instead of one fixed sleep; `ingestWaitMs` is the ceiling. */
215 readonly ingestPollMs: number
216 readonly warmupMs: number
217 readonly probe: {
218 readonly rps: number
219 readonly calls: readonly ProbeCall[]
220 /** Fraction of issued probes in (0, 1] that may be lost before the step is a harness fault. */
221 readonly maxLossRate: number
222 readonly onHarnessFault: ProbeFaultAction
223 readonly calibration: ProbeCalibration
224 /** Per-request deadline. A probe that hits it is `timed_out`, not a transport error. */
225 readonly timeoutMs: number
226 /** Outstanding probes. The next tick is `shed` once another batch would pass this. */
227 readonly maxInFlight: number
228 /** Issued probes required before the loss-rate gate can fail the step. */
229 readonly minSamples: number
230 /** A response at or above this, on both versions, is the same stall. */
231 readonly stallLatencyMs: number
232 /** How many dwells a harness stall may run, including the first. */
233 readonly windowAttempts: number
234 /** Wait between a stalled dwell and its re-run. */
235 readonly windowBackoffMs: number
236 readonly schedule: ProbeScheduleCalibration
237 }
238 readonly gates: RolloutGates
239 readonly burn: {
240 readonly fastBurn: number
241 readonly minSamples: number
242 }
243 readonly split: SplitPolicy
244 readonly telemetry: TelemetryPolicy
245 readonly cleanup: CleanupPolicy
246 readonly rate: RateGatePolicy
247 /** Evidence for the schedule's numbers (steps, ingest), keyed by field, like `probe.calibration`. Empty when omitted. */
248 readonly calibration: Readonly<Record<string, ScheduleCalibration>>
249}
251/** Citation for a schedule number: the runs it was read from, what they showed, and the headroom chosen. */
252export type ScheduleCalibration = {
253 readonly source: string
254 readonly date: string
255 readonly window: string
256 readonly n: number
257 readonly observed: string
258 readonly headroom: string
259}
261const SCHEDULE_FIELDS: readonly string[] = ['steps[*].percent', 'steps[*].dwellMs', 'ingestWaitMs', 'ingestPollMs', 'warmupMs']
263function scheduleCalibrationOf(value: unknown): Readonly<Record<string, ScheduleCalibration>> {
264 const rec = record(value, 'calibration')
265 const out: Record<string, ScheduleCalibration> = {}
266 for (const [key, entry] of Object.entries(rec)) {
267 const path = `calibration.${key}`
268 if (!SCHEDULE_FIELDS.includes(key)) reject(path, `names no schedule field (${SCHEDULE_FIELDS.join(', ')})`)
269 const row = record(entry, path)
270 exactKeys(row, path, ['source', 'date', 'window', 'n', 'observed', 'headroom'])
271 out[key] = {
272 source: textOf(row.source, `${path}.source`),
273 date: textOf(row.date, `${path}.date`),
274 window: textOf(row.window, `${path}.window`),
275 n: integer(row.n, `${path}.n`, 1),
276 observed: textOf(row.observed, `${path}.observed`),
277 headroom: textOf(row.headroom, `${path}.headroom`),
278 }
279 }
280 return out
281}
283export type LoadedPolicy = {
284 readonly policy: RolloutPolicy
285 readonly sha256: string
286 readonly body: Json
287}
289function reject(path: string, message: string): never {
290 throw new Error(`rollout policy ${path}: ${message}`)
291}
293function record(value: unknown, path: string): Record<string, unknown> {
294 if (value === null || typeof value !== 'object' || Array.isArray(value)) reject(path, 'must be an object')
295 return value as Record<string, unknown>
296}
298function exactKeys(rec: Record<string, unknown>, path: string, keys: readonly string[], optional: readonly string[] = []): void {
299 const allowed = new Set<string>([...keys, ...optional])
300 for (const key of Object.keys(rec)) {
301 if (!allowed.has(key)) reject(`${path}.${key}`, 'unknown field')
302 }
303 for (const key of keys) {
304 if (!(key in rec)) reject(`${path}.${key}`, 'missing')
305 }
306}
308function integer(value: unknown, path: string, min: number): number {
309 if (typeof value !== 'number' || !Number.isInteger(value) || value < min) reject(path, `must be an integer >= ${min}`)
310 return value
311}
313function positive(value: unknown, path: string): number {
314 if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) reject(path, 'must be a number > 0')
315 return value
316}
318function aboveOne(value: unknown, path: string): number {
319 if (typeof value !== 'number' || !Number.isFinite(value) || value <= 1) reject(path, 'must be a number > 1')
320 return value
321}
323function signal(value: unknown, path: string): SignalMode {
324 if (value !== 'hard' && value !== 'informational') reject(path, 'must be hard or informational')
325 return value
326}
328function oneOf<T extends string>(value: unknown, path: string, allowed: readonly T[]): T {
329 if (typeof value === 'string' && (allowed as readonly string[]).includes(value)) return value as T
330 reject(path, `must be ${allowed.join(' or ')}`)
331}
333function stepsOf(value: unknown): readonly RolloutStep[] {
334 if (!Array.isArray(value) || value.length === 0) reject('steps', 'must be a non-empty list')
335 const steps: RolloutStep[] = []
336 for (let index = 0; index < value.length; index++) {
337 const path = `steps[${index}]`
338 const rec = record(value[index], path)
339 exactKeys(rec, path, ['percent', 'dwellMs'])
340 const percent = integer(rec.percent, `${path}.percent`, 1)
341 if (percent > 100) reject(`${path}.percent`, 'must be <= 100')
342 const dwellMs = integer(rec.dwellMs, `${path}.dwellMs`, 1)
343 const previous = steps[steps.length - 1]
344 if (previous && percent <= previous.percent) reject(`${path}.percent`, 'steps must be strictly increasing')
345 steps.push({ percent, dwellMs })
346 }
347 const last = steps[steps.length - 1]
348 if (!last || last.percent !== 100) reject('steps', 'the last step must be 100')
349 // Ramp-only (Ivar, 2026-11-08): a lone 100 step is the removed direct promote, 0% straight to 100%.
350 if (steps.length < 2) reject('steps', 'must ramp: at least one step below 100 before the last (a single 0 to 100 step is a direct promote)')
351 return steps
352}
354/** `/path` plus an optional `?query`. The engine sends this string unchanged. */
355const PROBE_PATH = /^\/[^\s?#]*(\?[^\s#]*)?$/
357function probePath(value: unknown, path: string): string {
358 if (typeof value !== 'string' || !PROBE_PATH.test(value)) {
359 reject(path, 'must be a path starting with /, plus an optional query')
360 }
361 return value
362}
364function textOf(value: unknown, path: string): string {
365 if (typeof value !== 'string' || value.length === 0) reject(path, 'must be a non-empty string')
366 return value
367}
369function calibrationOf(value: unknown): ProbeCalibration {
370 const rec = record(value, 'probe.calibration')
371 exactKeys(rec, 'probe.calibration', ['source', 'date', 'window', 'n', 'lost', 'web', 'ply', 'observedRate', 'percentile', 'headroom'])
372 return {
373 source: textOf(rec.source, 'probe.calibration.source'),
374 date: textOf(rec.date, 'probe.calibration.date'),
375 window: textOf(rec.window, 'probe.calibration.window'),
376 n: integer(rec.n, 'probe.calibration.n', 1),
377 lost: integer(rec.lost, 'probe.calibration.lost', 0),
378 web: textOf(rec.web, 'probe.calibration.web'),
379 ply: textOf(rec.ply, 'probe.calibration.ply'),
380 observedRate: textOf(rec.observedRate, 'probe.calibration.observedRate'),
381 percentile: textOf(rec.percentile, 'probe.calibration.percentile'),
382 headroom: textOf(rec.headroom, 'probe.calibration.headroom'),
383 }
384}
386function scheduleOf(value: unknown): ProbeScheduleCalibration {
387 const rec = record(value, 'probe.schedule')
388 exactKeys(rec, 'probe.schedule', [
389 'source',
390 'date',
391 'window',
392 'n',
393 'responded',
394 'lost',
395 'healthyN',
396 'web',
397 'oldVersion',
398 'newVersion',
399 'healthyP50Ms',
400 'observedStallMs',
401 'dwellMs',
402 'observedRate',
403 'percentile',
404 'headroom',
405 ])
406 return {
407 source: textOf(rec.source, 'probe.schedule.source'),
408 date: textOf(rec.date, 'probe.schedule.date'),
409 window: textOf(rec.window, 'probe.schedule.window'),
410 n: integer(rec.n, 'probe.schedule.n', 1),
411 responded: integer(rec.responded, 'probe.schedule.responded', 0),
412 lost: integer(rec.lost, 'probe.schedule.lost', 0),
413 healthyN: integer(rec.healthyN, 'probe.schedule.healthyN', 1),
414 web: textOf(rec.web, 'probe.schedule.web'),
415 oldVersion: textOf(rec.oldVersion, 'probe.schedule.oldVersion'),
416 newVersion: textOf(rec.newVersion, 'probe.schedule.newVersion'),
417 healthyP50Ms: textOf(rec.healthyP50Ms, 'probe.schedule.healthyP50Ms'),
418 observedStallMs: integer(rec.observedStallMs, 'probe.schedule.observedStallMs', 0),
419 dwellMs: integer(rec.dwellMs, 'probe.schedule.dwellMs', 1),
420 observedRate: textOf(rec.observedRate, 'probe.schedule.observedRate'),
421 percentile: textOf(rec.percentile, 'probe.schedule.percentile'),
422 headroom: textOf(rec.headroom, 'probe.schedule.headroom'),
423 }
424}
426function callsOf(value: unknown): readonly ProbeCall[] {
427 if (!Array.isArray(value)) reject('probe.calls', 'must be a list')
428 return value.map((item, index) => {
429 const path = `probe.calls[${index}]`
430 const rec = record(item, path)
431 exactKeys(rec, path, ['method', 'path', 'auth'])
432 if (rec.method !== 'GET' && rec.method !== 'POST') reject(`${path}.method`, 'must be GET or POST')
433 const callPath = probePath(rec.path, `${path}.path`)
434 if (typeof rec.auth !== 'boolean') reject(`${path}.auth`, 'must be a boolean')
435 return { method: rec.method, path: callPath, auth: rec.auth }
436 })
437}
439function latencyOf(value: unknown): LatencyPolicy {
440 const rec = record(value, 'gates.latency')
441 const mode = rec.mode
442 if (mode === 'off' || mode === 'informational') {
443 exactKeys(rec, 'gates.latency', ['mode'])
444 return { mode }
445 }
446 if (mode !== 'gate') reject('gates.latency.mode', 'must be off, informational, or gate')
447 if (rec.minSamples === undefined || rec.p99Ratio === undefined || rec.effectFloorMs === undefined) {
448 reject('gates.latency', 'latency=gate needs minSamples, p99Ratio, and effectFloorMs')
449 }
450 exactKeys(rec, 'gates.latency', ['mode', 'minSamples', 'p99Ratio', 'effectFloorMs'])
451 return {
452 mode: 'gate',
453 minSamples: integer(rec.minSamples, 'gates.latency.minSamples', 1),
454 p99Ratio: aboveOne(rec.p99Ratio, 'gates.latency.p99Ratio'),
455 effectFloorMs: positive(rec.effectFloorMs, 'gates.latency.effectFloorMs'),
456 }
457}
459function gatesOf(value: unknown): RolloutGates {
460 const rec = record(value, 'gates')
461 exactKeys(rec, 'gates', ['smoke', 'fivexx', 'exceptions', 'errorLogs', 'nonOk', 'burn', 'latency'])
462 return {
463 smoke: signal(rec.smoke, 'gates.smoke'),
464 fivexx: signal(rec.fivexx, 'gates.fivexx'),
465 exceptions: signal(rec.exceptions, 'gates.exceptions'),
466 errorLogs: signal(rec.errorLogs, 'gates.errorLogs'),
467 nonOk: signal(rec.nonOk, 'gates.nonOk'),
468 burn: signal(rec.burn, 'gates.burn'),
469 latency: latencyOf(rec.latency),
470 }
471}
473function splitOf(value: unknown): SplitPolicy {
474 const rec = record(value, 'split')
475 exactKeys(rec, 'split', ['z', 'minNewRequests', 'onMismatch', 'onInsufficient'])
476 return {
477 z: positive(rec.z, 'split.z'),
478 minNewRequests: integer(rec.minNewRequests, 'split.minNewRequests', 1),
479 onMismatch: oneOf(rec.onMismatch, 'split.onMismatch', ['hold', 'rollback']),
480 onInsufficient: oneOf(rec.onInsufficient, 'split.onInsufficient', ['advance', 'hold']),
481 }
482}
484function rate(value: unknown, path: string): number {
485 if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0 || value > 1) reject(path, 'must be a number in (0, 1]')
486 return value
487}
489function cleanupCalibrationOf(value: unknown): CleanupCalibration {
490 const rec = record(value, 'cleanup.calibration')
491 exactKeys(rec, 'cleanup.calibration', ['source', 'date', 'window', 'n', 'latencyMs', 'web', 'namespace', 'observed', 'percentile', 'headroom'])
492 return {
493 source: textOf(rec.source, 'cleanup.calibration.source'),
494 date: textOf(rec.date, 'cleanup.calibration.date'),
495 window: textOf(rec.window, 'cleanup.calibration.window'),
496 n: integer(rec.n, 'cleanup.calibration.n', 1),
497 latencyMs: integer(rec.latencyMs, 'cleanup.calibration.latencyMs', 0),
498 web: textOf(rec.web, 'cleanup.calibration.web'),
499 namespace: textOf(rec.namespace, 'cleanup.calibration.namespace'),
500 observed: textOf(rec.observed, 'cleanup.calibration.observed'),
501 percentile: textOf(rec.percentile, 'cleanup.calibration.percentile'),
502 headroom: textOf(rec.headroom, 'cleanup.calibration.headroom'),
503 }
504}
506function cleanupOf(value: unknown): CleanupPolicy {
507 const rec = record(value, 'cleanup')
508 exactKeys(rec, 'cleanup', ['attempts', 'backoffMs', 'staleMinutes', 'calibration'])
509 return {
510 attempts: integer(rec.attempts, 'cleanup.attempts', 1),
511 backoffMs: integer(rec.backoffMs, 'cleanup.backoffMs', 0),
512 staleMinutes: integer(rec.staleMinutes, 'cleanup.staleMinutes', 1),
513 calibration: cleanupCalibrationOf(rec.calibration),
514 }
515}
517function queryCalibrationOf(value: unknown): QueryCalibration {
518 const rec = record(value, 'telemetry.query.calibration')
519 exactKeys(rec, 'telemetry.query.calibration', ['source', 'date', 'window', 'n', 'web', 'ply', 'observed', 'percentile', 'headroom'])
520 return {
521 source: textOf(rec.source, 'telemetry.query.calibration.source'),
522 date: textOf(rec.date, 'telemetry.query.calibration.date'),
523 window: textOf(rec.window, 'telemetry.query.calibration.window'),
524 n: integer(rec.n, 'telemetry.query.calibration.n', 1),
525 web: textOf(rec.web, 'telemetry.query.calibration.web'),
526 ply: textOf(rec.ply, 'telemetry.query.calibration.ply'),
527 observed: textOf(rec.observed, 'telemetry.query.calibration.observed'),
528 percentile: textOf(rec.percentile, 'telemetry.query.calibration.percentile'),
529 headroom: textOf(rec.headroom, 'telemetry.query.calibration.headroom'),
530 }
531}
533function queryPolicyOf(value: unknown): TelemetryQueryPolicy {
534 const rec = record(value, 'telemetry.query')
535 exactKeys(rec, 'telemetry.query', ['attempts', 'backoffMs', 'calibration'])
536 return {
537 attempts: integer(rec.attempts, 'telemetry.query.attempts', 1),
538 backoffMs: integer(rec.backoffMs, 'telemetry.query.backoffMs', 0),
539 calibration: queryCalibrationOf(rec.calibration),
540 }
541}
543/** Used only when the policy file omits `telemetry`. The numbers match the checked-in block. */
544function defaultTelemetry(): TelemetryPolicy {
545 return {
546 maxLossRate: DEFAULT_MAX_LOSS_RATE,
547 query: {
548 attempts: 3,
549 backoffMs: 1000,
550 calibration: {
551 source: 'run 2026-11-01T19-17-18-170Z',
552 date: '2026-11-01',
553 window: '100% ramp step',
554 n: 1,
555 web: '07568e0',
556 ply: '1743461',
557 observed: 'cf o11y telemetry query --body APIError timeout Envelope Error, exit 1',
558 percentile: 'one measured query, not a percentile',
559 headroom: 'default for an omitted telemetry block. attempts is 3. backoffMs 1000 is PRIOR: the run did not retry.',
560 },
561 },
562 }
563}
565function telemetryOf(value: unknown): TelemetryPolicy {
566 if (value === undefined) return defaultTelemetry()
567 const rec = record(value, 'telemetry')
568 exactKeys(rec, 'telemetry', ['query'], ['maxLossRate'])
569 return {
570 maxLossRate: rec.maxLossRate === undefined ? DEFAULT_MAX_LOSS_RATE : rate(rec.maxLossRate, 'telemetry.maxLossRate'),
571 query: queryPolicyOf(rec.query),
572 }
573}
575function unitRate(value: unknown, path: string): number {
576 if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) reject(path, 'must be a number in [0, 1]')
577 return value
578}
580function rateGateOf(parent: Record<string, unknown>): RateGatePolicy {
581 if (!('rate' in parent)) return DEFAULT_RATE_GATE
582 const rec = record(parent.rate, 'rate')
583 exactKeys(rec, 'rate', ['expectMode', 'probes', 'probeGapMs', 'probePath', 'minChecks', 'maxFailOpenRate', 'maxDroppedFlushes', 'calibration'])
584 const cal = record(rec.calibration, 'rate.calibration')
585 exactKeys(cal, 'rate.calibration', ['source', 'date', 'n', 'observed', 'headroom'])
586 const probes = integer(rec.probes, 'rate.probes', 0)
587 const minChecks = integer(rec.minChecks, 'rate.minChecks', 1)
588 if (probes > 0 && minChecks > probes) reject('rate.minChecks', 'must be <= rate.probes (the probes are what fills it)')
589 return {
590 expectMode: oneOf(rec.expectMode, 'rate.expectMode', ['batched', 'memo', 'legacy']),
591 probes,
592 probeGapMs: integer(rec.probeGapMs, 'rate.probeGapMs', 0),
593 probePath: probePath(rec.probePath, 'rate.probePath'),
594 minChecks,
595 maxFailOpenRate: unitRate(rec.maxFailOpenRate, 'rate.maxFailOpenRate'),
596 maxDroppedFlushes: integer(rec.maxDroppedFlushes, 'rate.maxDroppedFlushes', 0),
597 calibration: {
598 source: textOf(cal.source, 'rate.calibration.source'),
599 date: textOf(cal.date, 'rate.calibration.date'),
600 n: textOf(cal.n, 'rate.calibration.n'),
601 observed: textOf(cal.observed, 'rate.calibration.observed'),
602 headroom: textOf(cal.headroom, 'rate.calibration.headroom'),
603 },
604 }
605}
608function validate(value: unknown): RolloutPolicy {
609 const rec = record(value, 'policy')
610 // The direct promote (one write to 100%, `promote.watchMs` watch) was removed 2026-11-08 (Ivar: "Ramp should be the
611 // only option."). Name it, so an old policy file is not just an unknown key.
612 if ('promote' in rec) reject('promote', 'was removed 2026-11-08: the stepped ramp in steps is the only promotion path')
613 exactKeys(rec, 'policy', ['steps', 'ingestWaitMs', 'probe', 'gates', 'burn', 'split', 'cleanup'], ['warmupMs', 'telemetry', 'ingestPollMs', 'rate', 'calibration'])
614 const probe = record(rec.probe, 'probe')
615 exactKeys(probe, 'probe', [
616 'rps',
617 'calls',
618 'maxLossRate',
619 'onHarnessFault',
620 'calibration',
621 'timeoutMs',
622 'maxInFlight',
623 'minSamples',
624 'stallLatencyMs',
625 'windowAttempts',
626 'windowBackoffMs',
627 'schedule',
628 ])
629 const rps = probe.rps
630 if (typeof rps !== 'number' || !Number.isFinite(rps) || rps < 0) reject('probe.rps', 'must be a number >= 0')
631 const calls = callsOf(probe.calls)
632 if (rps > 0 && calls.length === 0) reject('probe.calls', 'probe rps needs at least one call')
633 const onHarnessFault = oneOf(probe.onHarnessFault, 'probe.onHarnessFault', ['rollback', 'hold', 'telemetry'])
634 const burn = record(rec.burn, 'burn')
635 exactKeys(burn, 'burn', ['fastBurn', 'minSamples'])
636 return {
637 steps: stepsOf(rec.steps),
638 ingestWaitMs: integer(rec.ingestWaitMs, 'ingestWaitMs', 0),
639 ingestPollMs: 'ingestPollMs' in rec ? integer(rec.ingestPollMs, 'ingestPollMs', 0) : 0,
640 warmupMs: rec.warmupMs === undefined ? 0 : integer(rec.warmupMs, 'warmupMs', 0),
641 probe: {
642 rps,
643 calls,
644 maxLossRate: rate(probe.maxLossRate, 'probe.maxLossRate'),
645 onHarnessFault,
646 calibration: calibrationOf(probe.calibration),
647 timeoutMs: integer(probe.timeoutMs, 'probe.timeoutMs', 1),
648 maxInFlight: integer(probe.maxInFlight, 'probe.maxInFlight', 1),
649 minSamples: integer(probe.minSamples, 'probe.minSamples', 1),
650 stallLatencyMs: integer(probe.stallLatencyMs, 'probe.stallLatencyMs', 1),
651 windowAttempts: integer(probe.windowAttempts, 'probe.windowAttempts', 1),
652 windowBackoffMs: integer(probe.windowBackoffMs, 'probe.windowBackoffMs', 0),
653 schedule: scheduleOf(probe.schedule),
654 },
655 gates: gatesOf(rec.gates),
656 burn: {
657 fastBurn: positive(burn.fastBurn, 'burn.fastBurn'),
658 minSamples: integer(burn.minSamples, 'burn.minSamples', 1),
659 },
660 split: splitOf(rec.split),
661 telemetry: telemetryOf(rec.telemetry),
662 cleanup: cleanupOf(rec.cleanup),
663 rate: rateGateOf(rec),
664 calibration: 'calibration' in rec ? scheduleCalibrationOf(rec.calibration) : {},
665 }
666}
668function policyBody(policy: RolloutPolicy): Json {
669 const latency: Json = policy.gates.latency.mode === 'gate'
670 ? {
671 mode: 'gate',
672 minSamples: policy.gates.latency.minSamples,
673 p99Ratio: policy.gates.latency.p99Ratio,
674 effectFloorMs: policy.gates.latency.effectFloorMs,
675 }
676 : { mode: policy.gates.latency.mode }
677 return {
678 steps: policy.steps.map(step => ({ percent: step.percent, dwellMs: step.dwellMs })),
679 ingestWaitMs: policy.ingestWaitMs,
680 ingestPollMs: policy.ingestPollMs,
681 warmupMs: policy.warmupMs,
682 probe: {
683 rps: policy.probe.rps,
684 maxLossRate: policy.probe.maxLossRate,
685 onHarnessFault: policy.probe.onHarnessFault,
686 calibration: {
687 source: policy.probe.calibration.source,
688 date: policy.probe.calibration.date,
689 window: policy.probe.calibration.window,
690 n: policy.probe.calibration.n,
691 lost: policy.probe.calibration.lost,
692 web: policy.probe.calibration.web,
693 ply: policy.probe.calibration.ply,
694 observedRate: policy.probe.calibration.observedRate,
695 percentile: policy.probe.calibration.percentile,
696 headroom: policy.probe.calibration.headroom,
697 },
698 calls: policy.probe.calls.map(call => ({ method: call.method, path: call.path, auth: call.auth })),
699 timeoutMs: policy.probe.timeoutMs,
700 maxInFlight: policy.probe.maxInFlight,
701 minSamples: policy.probe.minSamples,
702 stallLatencyMs: policy.probe.stallLatencyMs,
703 windowAttempts: policy.probe.windowAttempts,
704 windowBackoffMs: policy.probe.windowBackoffMs,
705 schedule: {
706 source: policy.probe.schedule.source,
707 date: policy.probe.schedule.date,
708 window: policy.probe.schedule.window,
709 n: policy.probe.schedule.n,
710 responded: policy.probe.schedule.responded,
711 lost: policy.probe.schedule.lost,
712 healthyN: policy.probe.schedule.healthyN,
713 web: policy.probe.schedule.web,
714 oldVersion: policy.probe.schedule.oldVersion,
715 newVersion: policy.probe.schedule.newVersion,
716 healthyP50Ms: policy.probe.schedule.healthyP50Ms,
717 observedStallMs: policy.probe.schedule.observedStallMs,
718 dwellMs: policy.probe.schedule.dwellMs,
719 observedRate: policy.probe.schedule.observedRate,
720 percentile: policy.probe.schedule.percentile,
721 headroom: policy.probe.schedule.headroom,
722 },
723 },
724 gates: {
725 smoke: policy.gates.smoke,
726 fivexx: policy.gates.fivexx,
727 exceptions: policy.gates.exceptions,
728 errorLogs: policy.gates.errorLogs,
729 nonOk: policy.gates.nonOk,
730 burn: policy.gates.burn,
731 latency,
732 },
733 burn: { fastBurn: policy.burn.fastBurn, minSamples: policy.burn.minSamples },
734 split: {
735 z: policy.split.z,
736 minNewRequests: policy.split.minNewRequests,
737 onMismatch: policy.split.onMismatch,
738 onInsufficient: policy.split.onInsufficient,
739 },
740 telemetry: {
741 maxLossRate: policy.telemetry.maxLossRate,
742 query: {
743 attempts: policy.telemetry.query.attempts,
744 backoffMs: policy.telemetry.query.backoffMs,
745 calibration: {
746 source: policy.telemetry.query.calibration.source,
747 date: policy.telemetry.query.calibration.date,
748 window: policy.telemetry.query.calibration.window,
749 n: policy.telemetry.query.calibration.n,
750 web: policy.telemetry.query.calibration.web,
751 ply: policy.telemetry.query.calibration.ply,
752 observed: policy.telemetry.query.calibration.observed,
753 percentile: policy.telemetry.query.calibration.percentile,
754 headroom: policy.telemetry.query.calibration.headroom,
755 },
756 },
757 },
758 calibration: Object.fromEntries(Object.entries(policy.calibration).map(([key, row]) => [key, {
759 source: row.source,
760 date: row.date,
761 window: row.window,
762 n: row.n,
763 observed: row.observed,
764 headroom: row.headroom,
765 }])),
766 rate: {
767 expectMode: policy.rate.expectMode,
768 probes: policy.rate.probes,
769 probeGapMs: policy.rate.probeGapMs,
770 probePath: policy.rate.probePath,
771 minChecks: policy.rate.minChecks,
772 maxFailOpenRate: policy.rate.maxFailOpenRate,
773 maxDroppedFlushes: policy.rate.maxDroppedFlushes,
774 calibration: { ...policy.rate.calibration },
775 },
776 cleanup: {
777 attempts: policy.cleanup.attempts,
778 backoffMs: policy.cleanup.backoffMs,
779 staleMinutes: policy.cleanup.staleMinutes,
780 calibration: {
781 source: policy.cleanup.calibration.source,
782 date: policy.cleanup.calibration.date,
783 window: policy.cleanup.calibration.window,
784 n: policy.cleanup.calibration.n,
785 latencyMs: policy.cleanup.calibration.latencyMs,
786 web: policy.cleanup.calibration.web,
787 namespace: policy.cleanup.calibration.namespace,
788 observed: policy.cleanup.calibration.observed,
789 percentile: policy.cleanup.calibration.percentile,
790 headroom: policy.cleanup.calibration.headroom,
791 },
792 },
793 }
794}
796/** Parse and validate. The hash is the raw file bytes, so two spellings of the same rules do not share a hash. */
797export function parsePolicy(text: string): LoadedPolicy {
798 let value: unknown
799 try {
800 value = JSON.parse(text) as unknown
801 } catch {
802 throw new Error('rollout policy is not JSON')
803 }
804 const policy = validate(value)
805 return { policy, sha256: createHash('sha256').update(text).digest('hex'), body: policyBody(policy) }
806}
808export function loadPolicy(path: string): LoadedPolicy {
809 let text: string
810 try {
811 text = readFileSync(path, 'utf8')
812 } catch {
813 throw new Error(`cannot read rollout policy ${path}`)
814 }
815 return parsePolicy(text)
816}
818/** `--policy` is removed before command parsing. A missing or repeated value is an error. */
819export function splitPolicy(argv: readonly string[]): { readonly argv: readonly string[]; readonly policyPath: string | null } | { readonly error: string } {
820 const kept: string[] = []
821 let policyPath: string | null = null
822 for (let index = 0; index < argv.length; index++) {
823 const arg = argv[index] ?? ''
824 if (arg !== '--policy') {
825 kept.push(arg)
826 continue
827 }
828 const value = argv[index + 1]
829 if (!value || value.startsWith('--')) return { error: '--policy needs a path' }
830 if (policyPath) return { error: '--policy was given twice' }
831 policyPath = value
832 index += 1
833 }
834 return { argv: kept, policyPath }
835}
Diagnostics4 git calls · 222 ms
CallStatusSizeTime0222 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit7 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=212 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=7 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2008.5 KiB212 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=8751 git.response.cache=MISS git.response.duration_ms=212 git.fetch.wants=0719c3dd2e202a01c260dc83fe2c2b834c6e3a91 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=8751 git.pkt.payload.max=8730 git.sideband.band1.bytes=8729 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2e85faf5decfa
- Request
- 17ad5ea5…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format