main
1/**
2 * SLO definitions and the ramp's burn-rate math. Numbers live here, not at the call site.
3 *
4 * Burn rate is the observed bad fraction divided by the error-budget fraction (1 − target).
5 * The ramp fails a step when that rate is above the policy's fast burn and the window
6 * has the policy's sample floor, or when a version (or the whole worker) has any
7 * exception, 5xx, error log, or non-ok outcome. Which of those signals actually
8 * stops a step comes from the rollout policy. Wall time is a gate only when that
9 * policy says `latency.mode` is `gate`. The routed share is a separate check.
10 * The multi-window table is the alerting policy for later; the step window uses only
11 * the fast-burn multiple.
12 */
14import type { RolloutPolicy } from './policy.ts'
16declare const ratioBrand: unique symbol
17/** A fraction strictly between 0 and 1. */
18export type Ratio = number & { readonly [ratioBrand]: 'Ratio' }
20declare const msBrand: unique symbol
21export type Milliseconds = number & { readonly [msBrand]: 'Milliseconds' }
23function ratio(value: number): Ratio {
24 if (!(value > 0 && value < 1)) throw new Error('SLO target must be between 0 and 1')
25 return value as Ratio
26}
28function milliseconds(value: number): Milliseconds {
29 if (!(value > 0)) throw new Error('ceiling must be positive')
30 return value as Milliseconds
31}
33export type SloId = 'availability' | 'read_latency' | 'write_latency' | 'correctness'
35export type Slo = {
36 readonly id: SloId
37 readonly statement: string
38 readonly target: Ratio
39 readonly windowDays: number
40 readonly initial: boolean
41 readonly ceilingMs: Milliseconds | null
42}
44export const READ_WALL_MS = milliseconds(1000)
45export const WRITE_WALL_MS = milliseconds(2500)
47export const AVAILABILITY: Slo = {
48 id: 'availability',
49 statement: 'good = non-5xx and no exception, over /v1 requests excluding expected 4xx',
50 target: ratio(0.999),
51 windowDays: 28,
52 initial: false,
53 ceilingMs: null,
54}
56/** Initial ceiling. Tighten from a real baseline; the target stays 99% under the ceiling. */
57export const READ_LATENCY: Slo = {
58 id: 'read_latency',
59 statement: 'GET /v1/* wall time under the ceiling',
60 target: ratio(0.99),
61 windowDays: 28,
62 initial: true,
63 ceilingMs: READ_WALL_MS,
64}
66/** Initial ceiling. Tighten from a real baseline; the target stays 99% under the ceiling. */
67export const WRITE_LATENCY: Slo = {
68 id: 'write_latency',
69 statement: 'POST /v1/* wall time under the ceiling',
70 target: ratio(0.99),
71 windowDays: 28,
72 initial: true,
73 ceilingMs: WRITE_WALL_MS,
74}
76export const CORRECTNESS: Slo = {
77 id: 'correctness',
78 statement: 'lifecycle smoke pass: every step expected status, fsck clean, refs equal',
79 target: ratio(0.995),
80 windowDays: 28,
81 initial: false,
82 ceilingMs: null,
83}
85export const SLOS: readonly Slo[] = [AVAILABILITY, READ_LATENCY, WRITE_LATENCY, CORRECTNESS]
87/**
88 * Google SRE workbook fast burn. On a 30-day budget, 14.4× for 1 hour spends about 2%.
89 * Our SLO window is 28 days; the multiple on a ramp step is still this constant.
90 */
91export const FAST_BURN = 14.4
93/**
94 * Sample floor copied into the checked-in policy as `burn.minSamples`.
95 * A ramp reads the policy. This export is the chart fallback for a run that
96 * recorded no policy event, and the lock that the file did not drift.
97 */
98export const MIN_SAMPLES = 100
100/**
101 * At 100% the old version is no longer allocated. The wall and CPU numbers on
102 * that line use the last step whose old version had at least this many wall
103 * samples. This is display, not a gate.
104 */
105export const CARRY_WALL_SAMPLES = 5
107/**
108 * Alerting policy for later. Not applied on a ramp step.
109 * The budget shares are the workbook's 30-day figures (1h/720, 6h/720, 3d/30d).
110 */
111export const BURN_ALERTS = [
112 { severity: 'page', multiple: 14.4, longWindow: '1h', shortWindow: '5m', budgetShare: 0.02 },
113 { severity: 'ticket', multiple: 6, longWindow: '6h', shortWindow: '30m', budgetShare: 0.05 },
114 { severity: 'ticket', multiple: 1, longWindow: '3d', shortWindow: '6h', budgetShare: 0.10 },
115] as const
117export type V1Request = {
118 readonly method: 'GET' | 'POST' | 'OTHER'
119 readonly path: string
120 readonly status: number
121 readonly exception: boolean
122 readonly wallMs: number | null
123 /** Ramp telemetry marks every 4xx expected, so client errors leave the availability denominator. */
124 readonly expected4xx: boolean
125}
127export type SloCount = {
128 readonly id: SloId
129 readonly good: number
130 readonly bad: number
131 readonly total: number
132 readonly burn: number | null
133}
135export type BurnMap = Readonly<Record<SloId, number | null>>
137export type AppliedRule = {
138 readonly name: string
139 readonly value: number | string
140}
142export type StepVerdict = {
143 readonly outcome: 'advance' | 'rollback' | 'hold' | 'insufficient_data'
144 readonly reason: string
145 readonly rules: readonly [AppliedRule, ...AppliedRule[]]
146 readonly burns: BurnMap
147}
149export function errorBudget(target: Ratio): number {
150 return 1 - target
151}
153/** Null when the window saw nothing. Otherwise bad/total divided by the error budget. */
154export function burnRate(bad: number, total: number, target: Ratio): number | null {
155 if (total <= 0) return null
156 return (bad / total) / errorBudget(target)
157}
159function isV1(path: string): boolean {
160 return path === '/v1' || path.startsWith('/v1/')
161}
163function skippedClientError(request: V1Request): boolean {
164 return request.expected4xx && request.status >= 400 && request.status < 500 && !request.exception
165}
167export function countSlo(slo: Slo, requests: readonly V1Request[]): SloCount {
168 let good = 0
169 let bad = 0
170 for (const request of requests) {
171 if (!isV1(request.path)) continue
172 if (slo.id === 'availability') {
173 if (skippedClientError(request)) continue
174 if (request.exception || request.status >= 500) bad += 1
175 else good += 1
176 continue
177 }
178 if (slo.id === 'read_latency' && request.method !== 'GET') continue
179 if (slo.id === 'write_latency' && request.method !== 'POST') continue
180 if (slo.id !== 'read_latency' && slo.id !== 'write_latency') continue
181 if (request.wallMs === null || slo.ceilingMs === null) continue
182 if (request.wallMs < slo.ceilingMs) good += 1
183 else bad += 1
184 }
185 const total = good + bad
186 return { id: slo.id, good, bad, total, burn: burnRate(bad, total, slo.target) }
187}
189/** One lifecycle run, or the steps inside it. The smoke stage still fails closed on any failed step. */
190export function countCorrectness(passed: number, failed: number): SloCount {
191 const total = passed + failed
192 return {
193 id: CORRECTNESS.id,
194 good: passed,
195 bad: failed,
196 total,
197 burn: burnRate(failed, total, CORRECTNESS.target),
198 }
199}
201export function burnMap(requests: readonly V1Request[], correctness: SloCount | null = null): BurnMap {
202 return {
203 availability: countSlo(AVAILABILITY, requests).burn,
204 read_latency: countSlo(READ_LATENCY, requests).burn,
205 write_latency: countSlo(WRITE_LATENCY, requests).burn,
206 correctness: correctness?.burn ?? null,
207 }
208}
210export type StepInput = {
211 readonly requests: number
212 /** External fetch invocations. The routed share uses this, not `requests`, which falls back to logs. */
213 readonly invocations: number
214 readonly exceptions: number
215 readonly errors: number
216 readonly fivexx: number
217 readonly errorLogs: number
218 readonly wallMs: readonly number[]
219 readonly v1: readonly V1Request[]
220}
222export type RoutedShare = {
223 readonly n: number
224 readonly newRequests: number
225 readonly share: number | null
226 readonly low: number | null
227 readonly high: number | null
228 readonly kind: 'untested' | 'match' | 'mismatch'
229}
231/**
232 * Binomial band around the target percent. Width is `z` standard errors at this n.
233 * `untested` is n = 0. `mismatch` is a share outside the band, or a new count that
234 * is not a fraction of the aggregate.
235 */
236export function routedShare(newRequests: number, total: number, targetPercent: number, z: number): RoutedShare {
237 if (!(total > 0) || newRequests < 0 || newRequests > total) {
238 const kind = total >= 0 && newRequests > total ? 'mismatch' : 'untested'
239 return { n: Math.max(0, total), newRequests, share: null, low: null, high: null, kind }
240 }
241 const p = targetPercent / 100
242 const se = Math.sqrt((p * (1 - p)) / total)
243 const margin = z * se
244 const low = Math.max(0, p - margin)
245 const high = Math.min(1, p + margin)
246 const share = newRequests / total
247 const kind = share < low || share > high ? 'mismatch' : 'match'
248 return { n: total, newRequests, share, low, high, kind }
249}
251function rulesOf(list: readonly AppliedRule[]): readonly [AppliedRule, ...AppliedRule[]] {
252 const first = list[0]
253 if (!first) throw new Error('a decision needs a rule')
254 return [first, ...list.slice(1)]
255}
257function verdict(
258 outcome: StepVerdict['outcome'],
259 reason: string,
260 rules: readonly AppliedRule[],
261 burns: BurnMap,
262): StepVerdict {
263 return { outcome, reason, rules: rulesOf(rules), burns }
264}
266function percentile(values: readonly number[], p: number): number | null {
267 const sorted = values.filter(value => Number.isFinite(value)).slice().sort((a, b) => a - b)
268 if (sorted.length === 0) return null
269 return sorted[Math.floor((sorted.length - 1) * p)] ?? null
270}
272type Breach = { readonly reason: string; readonly rule: AppliedRule }
274/** Hard breaches stop the step. Informational ones are named on the advance reason. */
275function consider(mode: 'hard' | 'informational', breach: Breach | null, hard: Breach[], notes: string[]): void {
276 if (!breach) return
277 if (mode === 'hard') hard.push(breach)
278 else notes.push(breach.reason)
279}
281function countBreach(count: number, noun: string, rule: string, scope: string): Breach | null {
282 if (count <= 0) return null
283 const word = count === 1 ? noun : `${noun}s`
284 return { reason: `${count} ${word} ${scope}`, rule: { name: rule, value: 0 } }
285}
287function points(fraction: number): string {
288 return (fraction * 100).toFixed(2)
289}
291/**
292 * Version-vs-version wall p99. Fires only for `latency.mode = gate`, and only
293 * once both sides have the policy's minSamples. A short sample does not roll back.
294 */
295function latencyBreach(newer: readonly number[], older: readonly number[], policy: RolloutPolicy): Breach | null {
296 const latency = policy.gates.latency
297 if (latency.mode !== 'gate') return null
298 if (newer.length < latency.minSamples || older.length < latency.minSamples) return null
299 const next = percentile(newer, 0.99)
300 const previous = percentile(older, 0.99)
301 if (next === null || previous === null) return null
302 const delta = next - previous
303 const floor = { name: 'latency.effectFloorMs', value: latency.effectFloorMs }
304 if (previous <= 0) {
305 if (delta <= latency.effectFloorMs) return null
306 return { reason: `wall p99 was ${previous} on the old version and ${next} on the new`, rule: floor }
307 }
308 const ratio = next / previous
309 if (ratio > latency.p99Ratio && delta > latency.effectFloorMs) {
310 return {
311 reason: `wall p99 ratio ${ratio.toFixed(3)} > ${latency.p99Ratio} and +${delta}ms > ${latency.effectFloorMs}ms`,
312 rule: { name: 'latency.p99Ratio', value: latency.p99Ratio },
313 }
314 }
315 return null
316}
318const SCOPES = [
319 { id: 'new', scope: 'on the new version', pick: (input: { readonly next: StepInput }) => input.next },
320 { id: 'old', scope: 'on the old version', pick: (input: { readonly previous: StepInput }) => input.previous },
321 { id: 'aggregate', scope: 'across the worker', pick: (input: { readonly aggregate: StepInput }) => input.aggregate },
322] as const
324/**
325 * One ramp step. Correctness and burn are judged per version and across the worker.
326 * The routed share uses external fetch invocations. Too few new-version requests is
327 * `insufficient_data` or an advance, per the policy. A share outside the binomial
328 * band holds or rolls back, per the policy. Latency rolls back only in `gate` mode.
329 */
330export function judgeStep(input: {
331 readonly next: StepInput
332 readonly previous: StepInput
333 readonly aggregate: StepInput
334 readonly targetPercent: number
335 readonly policy: RolloutPolicy
336}): StepVerdict {
337 const burns = burnMap(input.next.v1)
338 const hard: Breach[] = []
339 const notes: string[] = []
340 const gates = input.policy.gates
341 for (const scope of SCOPES) {
342 const side = scope.pick(input)
343 // Only the new version's own signals can fail its release. The old version (and the aggregate, which is the
344 // old version plus the new one) is informational: run 2026-11-06T20-57-44-803Z rolled back a clean new version
345 // for "1 5xx response on the old version". A new-version breach is already counted on the `new` scope.
346 const own = (mode: 'hard' | 'informational'): 'hard' | 'informational' => scope.id === 'new' ? mode : 'informational'
347 consider(own(gates.exceptions), countBreach(side.exceptions, 'exception', `max_${scope.id}_exceptions`, scope.scope), hard, notes)
348 consider(own(gates.fivexx), countBreach(side.fivexx, '5xx response', `max_${scope.id}_5xx`, scope.scope), hard, notes)
349 consider(own(gates.errorLogs), countBreach(side.errorLogs, 'error log', `max_${scope.id}_error_logs`, scope.scope), hard, notes)
350 consider(own(gates.nonOk), countBreach(side.errors, 'non-ok outcome', `max_${scope.id}_errors`, scope.scope), hard, notes)
351 if (gates.burn !== 'hard' && gates.burn !== 'informational') continue
352 for (const slo of [AVAILABILITY, READ_LATENCY, WRITE_LATENCY]) {
353 const count = countSlo(slo, side.v1)
354 if (count.total < input.policy.burn.minSamples || count.burn === null || count.burn <= input.policy.burn.fastBurn) continue
355 consider(
356 own(gates.burn),
357 {
358 reason: `${slo.id} burn ${count.burn.toFixed(2)} > ${input.policy.burn.fastBurn} ${scope.scope}`,
359 rule: { name: 'fast_burn', value: input.policy.burn.fastBurn },
360 },
361 hard,
362 notes,
363 )
364 }
365 }
366 consider(gates.latency.mode === 'gate' ? 'hard' : 'informational', latencyBreach(input.next.wallMs, input.previous.wallMs, input.policy), hard, notes)
367 const first = hard[0]
368 if (first) return verdict('rollback', first.reason, [first.rule], burns)
369 const routed = routedShare(input.next.invocations, input.aggregate.invocations, input.targetPercent, input.policy.split.z)
370 // At 0% or 100% the binomial band has zero width, so any request still served by the other version while the
371 // split propagates is a "mismatch" (runs 2026-11-06T20-01-37-842Z and 20-11-49-685Z: 98.84% and 98.74% at 100%,
372 // both rolled back a clean version). There it is informational; the live smoke checks x-ply-version.
373 const degenerate = input.targetPercent <= 0 || input.targetPercent >= 100
374 if (routed.kind === 'mismatch' && degenerate && routed.share !== null) {
375 notes.push(`routed share ${points(routed.share)}% at target ${input.targetPercent}% (n=${routed.n}): split propagation`)
376 } else if (routed.kind === 'mismatch') {
377 const where = routed.share === null
378 ? 'new-version requests are not a fraction of the aggregate'
379 : `routed share ${points(routed.share)}% is outside ${points(routed.low ?? 0)}..${points(routed.high ?? 0)} of target ${input.targetPercent}% (n=${routed.n})`
380 const outcome = input.policy.split.onMismatch === 'rollback' ? 'rollback' : 'hold'
381 return verdict(outcome, where, [{ name: 'routed_share', value: input.policy.split.z }], burns)
382 }
383 const noted = notes.length === 0 ? '' : `; informational: ${notes.join('; ')}`
384 if (routed.kind === 'untested' || routed.newRequests < input.policy.split.minNewRequests) {
385 const reason = routed.kind === 'untested'
386 ? `insufficient evidence: 0 requests in the window${noted}`
387 : `insufficient evidence: ${routed.newRequests} new-version requests < ${input.policy.split.minNewRequests}${noted}`
388 const rule = { name: 'min_new_requests', value: input.policy.split.minNewRequests }
389 if (input.policy.split.onInsufficient === 'hold') return verdict('insufficient_data', reason, [rule], burns)
390 return verdict('advance', reason, [rule], burns)
391 }
392 const reason = notes.length === 0
393 ? `within burn ${input.policy.burn.fastBurn}`
394 : `within burn ${input.policy.burn.fastBurn}${noted}`
395 return verdict('advance', reason, [{ name: 'fast_burn', value: input.policy.burn.fastBurn }], burns)
396}
Diagnostics4 git calls · 274 ms
CallStatusSizeTime0274 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2005.2 KiB268 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=268 git.response.bytes=5301 git.response.cache=MISS git.response.duration_ms=268 git.fetch.wants=fcba12fb7df2b6268338f877395e2345f2f04ca2 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5301 git.pkt.payload.max=5280 git.sideband.band1.bytes=5279 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2ccf74c9a7929
- Request
- 3ae8452a…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format