main
1import assert from 'node:assert/strict'
2import { readFileSync } from 'node:fs'
3import { test } from 'node:test'
4import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
5import {
6 AVAILABILITY,
7 burnRate,
8 countCorrectness,
9 countSlo,
10 errorBudget,
11 FAST_BURN,
12 judgeStep,
13 MIN_SAMPLES,
14 READ_LATENCY,
15 routedShare,
16 type StepInput,
17 type V1Request,
18} from '../scripts/cd/slo.ts'
20function request(over: Partial<V1Request> = {}): V1Request {
21 return {
22 method: 'GET',
23 path: '/v1/discover',
24 status: 200,
25 exception: false,
26 wallMs: 100,
27 expected4xx: false,
28 ...over,
29 }
30}
32function side(over: Partial<StepInput> = {}): StepInput {
33 const row = {
34 requests: 0,
35 invocations: 0,
36 exceptions: 0,
37 errors: 0,
38 fivexx: 0,
39 errorLogs: 0,
40 wallMs: [] as readonly number[],
41 v1: [] as readonly V1Request[],
42 ...over,
43 }
44 return { ...row, invocations: over.invocations ?? row.requests }
45}
47function judge(next: StepInput, previous: StepInput, policy: RolloutPolicy, targetPercent = 50, aggregate?: StepInput) {
48 return judgeStep({
49 next,
50 previous,
51 aggregate: aggregate ?? side({
52 requests: next.requests + previous.requests,
53 invocations: next.invocations + previous.invocations,
54 exceptions: next.exceptions + previous.exceptions,
55 errors: next.errors + previous.errors,
56 fivexx: next.fivexx + previous.fivexx,
57 errorLogs: next.errorLogs + previous.errorLogs,
58 wallMs: [...next.wallMs, ...previous.wallMs],
59 v1: [...next.v1, ...previous.v1],
60 }),
61 targetPercent,
62 policy,
63 })
64}
66function checkedPolicy(patch?: (raw: Record<string, unknown>) => void): RolloutPolicy {
67 const raw = JSON.parse(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')) as Record<string, unknown>
68 patch?.(raw)
69 return parsePolicy(JSON.stringify(raw)).policy
70}
72function walls(n: number, value: number): number[] {
73 return Array.from({ length: n }, () => value)
74}
76test('burn rate is the bad fraction divided by the error budget', () => {
77 assert.ok(Math.abs(errorBudget(AVAILABILITY.target) - 0.001) < 1e-12)
78 assert.ok(Math.abs((burnRate(1, 1000, AVAILABILITY.target) ?? 0) - 1) < 1e-9)
79 assert.equal(burnRate(0, 10, READ_LATENCY.target), 0)
80 assert.equal(burnRate(0, 0, READ_LATENCY.target), null)
81 assert.ok((burnRate(3, 20, READ_LATENCY.target) ?? 0) > FAST_BURN)
82 assert.ok((burnRate(2, 20, READ_LATENCY.target) ?? 0) < FAST_BURN)
83})
85test('expected 4xx leave the availability denominator and a slow read is a latency miss', () => {
86 const requests = [
87 request({ status: 404, expected4xx: true }),
88 request({ status: 500 }),
89 request({ status: 200 }),
90 request({ method: 'POST', path: '/v1/edit', wallMs: 3000 }),
91 request({ wallMs: 1500 }),
92 ]
93 const availability = countSlo(AVAILABILITY, requests)
94 assert.equal(availability.total, 4)
95 assert.equal(availability.bad, 1)
96 const reads = countSlo(READ_LATENCY, requests)
97 assert.equal(reads.bad, 1)
98 assert.equal(reads.good, 3)
99 const correctness = countCorrectness(199, 1)
100 assert.ok(Math.abs((correctness.burn ?? 0) - 1) < 1e-9)
101})
103test('a short clean sample is insufficient evidence and a fast burn rolls back', () => {
104 const policy = checkedPolicy()
105 const quiet = judge(side({ requests: 3, wallMs: walls(3, 80) }), side({ requests: 4, wallMs: walls(4, 70) }), policy)
106 assert.equal(quiet.outcome, 'insufficient_data')
107 assert.equal(quiet.rules[0]?.name, 'min_new_requests')
108 const burned = judge(
109 side({
110 requests: MIN_SAMPLES,
111 wallMs: walls(MIN_SAMPLES, 80),
112 v1: walls(MIN_SAMPLES, 1500).map(wallMs => request({ wallMs })),
113 }),
114 side({ requests: MIN_SAMPLES, wallMs: walls(MIN_SAMPLES, 80) }),
115 policy,
116 )
117 assert.equal(burned.outcome, 'rollback')
118 assert.equal(burned.rules[0]?.name, 'fast_burn')
119 assert.equal(burned.rules[0]?.value, FAST_BURN)
120 assert.match(burned.reason, /read_latency burn/)
121 assert.match(burned.reason, /on the new version/)
122})
124test('any exception, 5xx, error log, or non-ok outcome rolls back before the sample floor', () => {
125 const policy = checkedPolicy()
126 const exception = judge(side({ requests: 1, exceptions: 1, errors: 1 }), side({ requests: 40 }), policy)
127 assert.deepEqual(exception.outcome, 'rollback')
128 assert.equal(exception.reason, '1 exception on the new version')
129 const fivexx = judge(side({ requests: 1, fivexx: 1 }), side({ requests: 1 }), policy)
130 assert.match(fivexx.reason, /5xx/)
131 assert.equal(fivexx.outcome, 'rollback')
132 const logged = judge(side({ requests: 1, errorLogs: 1 }), side({ requests: 4 }), policy)
133 assert.equal(logged.outcome, 'rollback')
134 assert.equal(logged.reason, '1 error log on the new version')
135})
137// Run 2026-11-06T20-57-44-803Z rolled a clean new version back for "1 5xx response on the old version".
138test('the old version and the aggregate never fail the new release; they are named as informational', () => {
139 const policy = checkedPolicy()
140 const old = judge(side({ requests: MIN_SAMPLES }), side({ requests: MIN_SAMPLES, fivexx: 1, exceptions: 1, errorLogs: 1, errors: 1 }), policy)
141 assert.equal(old.outcome, 'advance')
142 assert.match(old.reason, /informational: .*1 5xx response on the old version/)
143 const across = judge(
144 side({ requests: MIN_SAMPLES }),
145 side({ requests: MIN_SAMPLES }),
146 policy,
147 50,
148 side({ requests: MIN_SAMPLES * 2, fivexx: 1 }),
149 )
150 assert.equal(across.outcome, 'advance')
151 assert.match(across.reason, /informational: .*1 5xx response across the worker/)
152 const newer = judge(side({ requests: MIN_SAMPLES, fivexx: 1 }), side({ requests: MIN_SAMPLES, fivexx: 3 }), policy)
153 assert.equal(newer.outcome, 'rollback')
154 assert.equal(newer.reason, '1 5xx response on the new version')
155})
157test('the checked-in policy does not roll back on wall time', () => {
158 const policy = checkedPolicy()
159 const fixture = JSON.parse(readFileSync(new URL('./fixtures/cd/ramp-5pct-walls.json', import.meta.url), 'utf8')) as {
160 next: number[]
161 previous: number[]
162 }
163 const recorded = judge(
164 side({ requests: fixture.next.length, wallMs: fixture.next }),
165 side({ requests: fixture.previous.length, wallMs: fixture.previous }),
166 policy,
167 )
168 assert.notEqual(recorded.outcome, 'rollback')
169 const wide = judge(side({ requests: 161, wallMs: walls(161, 27) }), side({ requests: 161, wallMs: walls(161, 12) }), policy)
170 assert.equal(wide.outcome, 'advance')
171 const gross = judge(
172 side({ requests: MIN_SAMPLES, wallMs: walls(MIN_SAMPLES, 210) }),
173 side({ requests: MIN_SAMPLES, wallMs: walls(MIN_SAMPLES, 80) }),
174 policy,
175 )
176 assert.equal(gross.outcome, 'advance')
177})
179test('latency=gate rolls back only when the policy thresholds clear', () => {
180 const policy = checkedPolicy(raw => {
181 raw.gates = {
182 ...(raw.gates as Record<string, unknown>),
183 latency: { mode: 'gate', minSamples: MIN_SAMPLES, p99Ratio: 2, effectFloorMs: 10 },
184 }
185 })
186 const clear = judge(
187 side({ requests: MIN_SAMPLES, wallMs: walls(MIN_SAMPLES, 80) }),
188 side({ requests: MIN_SAMPLES, wallMs: walls(MIN_SAMPLES, 20) }),
189 policy,
190 )
191 assert.equal(clear.outcome, 'rollback')
192 assert.match(clear.reason, /wall p99 ratio 4.000 > 2/)
193 assert.match(clear.reason, /\+60ms > 10ms/)
194 const short = judge(side({ requests: 6, wallMs: walls(6, 210) }), side({ requests: 6, wallMs: walls(6, 80) }), policy)
195 assert.notEqual(short.outcome, 'rollback')
196 const fixture = JSON.parse(readFileSync(new URL('./fixtures/cd/ramp-5pct-walls.json', import.meta.url), 'utf8')) as {
197 next: number[]
198 previous: number[]
199 }
200 const recorded = judge(
201 side({ requests: fixture.next.length, wallMs: fixture.next }),
202 side({ requests: fixture.previous.length, wallMs: fixture.previous }),
203 policy,
204 )
205 assert.notEqual(recorded.outcome, 'rollback')
206})
208test('an informational signal is recorded and does not roll back', () => {
209 const policy = checkedPolicy(raw => {
210 raw.gates = { ...(raw.gates as Record<string, unknown>), fivexx: 'informational' }
211 raw.split = { ...(raw.split as Record<string, unknown>), minNewRequests: 1 }
212 })
213 const step = judge(side({ requests: 2, fivexx: 1 }), side({ requests: 2 }), policy)
214 assert.equal(step.outcome, 'advance')
215 assert.match(step.reason, /informational: 1 5xx response on the new version/)
216})
218test('the routed share matches a binomial band, and a thin sample is typed', () => {
219 const policy = checkedPolicy()
220 const fixture = JSON.parse(readFileSync(new URL('./fixtures/cd/routed-share.json', import.meta.url), 'utf8')) as {
221 targetPercent: number
222 within: { newRequests: number; total: number }
223 mismatch: { newRequests: number; total: number }
224 thin: { newRequests: number; total: number }
225 }
226 const at = (counts: { newRequests: number; total: number }) => judge(
227 side({ requests: counts.newRequests, invocations: counts.newRequests }),
228 side({ requests: counts.total - counts.newRequests, invocations: counts.total - counts.newRequests }),
229 policy,
230 fixture.targetPercent,
231 side({ requests: counts.total, invocations: counts.total }),
232 )
233 const within = at(fixture.within)
234 assert.equal(within.outcome, 'advance')
235 const missed = at(fixture.mismatch)
236 assert.equal(missed.outcome, 'rollback')
237 assert.equal(missed.rules[0]?.name, 'routed_share')
238 assert.match(missed.reason, /outside/)
239 const thin = at(fixture.thin)
240 assert.equal(thin.outcome, 'insufficient_data')
241 assert.match(thin.reason, /40 new-version requests < 100/)
242 const band = routedShare(200, 2000, 10, 3)
243 assert.equal(band.kind, 'match')
244 assert.ok(Math.abs((band.low ?? 0) * 100 - 7.99) < 0.02)
245 assert.ok(Math.abs((band.high ?? 0) * 100 - 12.01) < 0.02)
246 const held = checkedPolicy(raw => {
247 raw.split = { ...(raw.split as Record<string, unknown>), onMismatch: 'hold' }
248 })
249 assert.equal(judge(
250 side({ requests: 1000, invocations: 1000 }),
251 side({ requests: 1000, invocations: 1000 }),
252 held,
253 10,
254 side({ requests: 2000, invocations: 2000 }),
255 ).outcome, 'hold')
256 const advanceThin = checkedPolicy(raw => {
257 raw.split = { ...(raw.split as Record<string, unknown>), onInsufficient: 'advance' }
258 })
259 const moved = judge(
260 side({ requests: 40, invocations: 40 }),
261 side({ requests: 360, invocations: 360 }),
262 advanceThin,
263 10,
264 side({ requests: 400, invocations: 400 }),
265 )
266 assert.equal(moved.outcome, 'advance')
267 assert.match(moved.reason, /insufficient evidence/)
268})
Diagnostics4 git calls · 42 ms
CallStatusSizeTime042 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2002.7 KiB · hit37 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=37 git.response.bytes=2744 git.response.cache=HIT git.response.duration_ms=37 git.fetch.wants=75c0fcaeb11a478a0717eb8fa270b21e6ad05c5e git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2744 git.pkt.payload.max=2723 git.sideband.band1.bytes=2722 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e301b038397929
- Request
- 5b3b4fc0…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format