main
1import assert from 'node:assert/strict'
2import { readFileSync } from 'node:fs'
3import { test } from 'node:test'
4import { none, some } from '../src/option.ts'
5import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
6import { accountProbeLoss, classifyProbe, judgeProbeWindow, type ProbeJudge, type ProbeOutcome } from '../scripts/cd/probe.ts'
7import { runRamp, type Observation } from '../scripts/cd/ramp.ts'
8import type { VersionStats } from '../scripts/cd/telemetry.ts'
10function policy(over: { readonly onHarnessFault: 'rollback' | 'hold'; readonly maxLossRate: number; readonly minSamples: number } | null = null): RolloutPolicy {
11 const raw = JSON.parse(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')) as {
12 steps: { percent: number; dwellMs: number }[]
13 probe: { maxLossRate: number; onHarnessFault: string }
14 burn: { minSamples: number }
15 ingestPollMs: number
16 }
17 raw.steps = [{ percent: 75, dwellMs: 1 }, { percent: 100, dwellMs: 1 }]
18 // These tests drive the probe-loss gate's traffic modes with a frozen clock: the fixed ingest sleep (a polled wait
19 // needs a clock that moves), and onHarnessFault rollback unless a test picks another (the file is telemetry since
20 // 2026-11-08; test/cd-ramp-only.test.ts covers that mode).
21 raw.ingestPollMs = 0
22 raw.probe.onHarnessFault = 'rollback'
23 if (over) {
24 raw.probe.maxLossRate = over.maxLossRate
25 raw.probe.onHarnessFault = over.onHarnessFault
26 raw.burn.minSamples = over.minSamples
27 }
28 return parsePolicy(JSON.stringify(raw)).policy
29}
31function judgeOf(maxLossRate: number, respondedMin: number, minSamples: number): ProbeJudge {
32 return { maxLossRate, respondedMin, minSamples, stallLatencyMs: 1_000, fivexxHard: true }
33}
35function responded(id: string, status = 200): ProbeOutcome {
36 return classifyProbe(id, { kind: 'response', status, ray: some('ray'), latencyMs: 50, version: none() })
37}
39function aborted(id: string): ProbeOutcome {
40 return classifyProbe(id, { kind: 'thrown', name: 'AbortError', message: 'The operation was aborted' })
41}
43function fill(count: number, status = 200): ProbeOutcome[] {
44 const outcomes: ProbeOutcome[] = []
45 for (let index = 0; index < count; index++) outcomes.push(responded(`ok-${index}`, status))
46 return outcomes
47}
49function stats(requests: number): VersionStats {
50 return {
51 requests,
52 invocations: requests,
53 exceptions: 0,
54 errors: 0,
55 fivexx: 0,
56 errorLogs: 0,
57 wallMs: [10, 10, 10, 10, 10],
58 cpuMs: [10],
59 fetchMs: [10],
60 v1: [],
61 internal: [],
62 }
63}
65function windowFor(percent: number): Observation {
66 if (percent >= 100) {
67 const next = stats(200)
68 const previous = stats(0)
69 return { kind: 'sample', next, previous: { ...previous, wallMs: [], cpuMs: [], fetchMs: [] }, aggregate: next, landed: next.invocations }
70 }
71 const next = stats(150)
72 const previous = stats(50)
73 const aggregate = stats(200)
74 return { kind: 'sample', next, previous, aggregate, landed: aggregate.invocations }
75}
77test('a client abort with no status and no ray is a loss, and a 500 is a responded server error', () => {
78 const lost = aborted('1 GET https://ply.ivar.workers.dev/')
79 assert.equal(lost.kind, 'lost')
80 if (lost.kind === 'lost') assert.equal(lost.reason, 'aborted')
81 const timed = classifyProbe('timed', { kind: 'timeout' })
82 assert.deepEqual(timed, { kind: 'lost', id: 'timed', reason: 'timed_out' })
83 const named = classifyProbe('named', { kind: 'thrown', name: 'TimeoutError', message: 'The operation was aborted due to timeout' })
84 assert.deepEqual(named, { kind: 'lost', id: 'named', reason: 'timed_out' })
85 const network = classifyProbe('net', { kind: 'thrown', name: 'TypeError', message: 'fetch failed' })
86 assert.deepEqual(network, { kind: 'lost', id: 'net', reason: 'network' })
87 const empty = classifyProbe('empty', { kind: 'response', status: 0, ray: some('ray'), latencyMs: 50, version: none() })
88 assert.equal(empty.kind, 'lost')
89 const server = classifyProbe('srv', { kind: 'response', status: 500, ray: some('ray'), latencyMs: 50, version: none() })
90 assert.equal(server.kind, 'responded')
91 const loss = accountProbeLoss([lost, server, ...fill(1)], 1 / 790)
92 assert.equal(loss.count, 1)
93 assert.equal(loss.responded, 2)
94 assert.equal(loss.serverErrors, 1)
95 assert.deepEqual(loss.ids, ['1 GET https://ply.ivar.workers.dev/'])
96 assert.equal(loss.exceeded, true)
97})
99test('the measured 1/790 abort is under the checked-in rate and is not a server error', () => {
100 const outcomes = [...fill(789), aborted('790 GET https://ply.ivar.workers.dev/')]
101 const loss = accountProbeLoss(outcomes, 1 / 790)
102 assert.equal(loss.total, 790)
103 assert.equal(loss.count, 1)
104 assert.equal(loss.rate, 1 / 790)
105 assert.equal(loss.exceeded, false)
106 assert.equal(loss.responded, 789)
107 assert.equal(loss.serverErrors, 0)
108 const window = judgeProbeWindow(outcomes, judgeOf(1 / 790, 100, 1_000))
109 assert.equal(window.kind, 'ok')
110})
112test('one more loss than the measured rate is a harness fault once the denominator is large enough', () => {
113 const under = judgeProbeWindow([...fill(788), aborted('a'), aborted('b')], judgeOf(1 / 790, 100, 1_000))
114 assert.notEqual(under.kind, 'harness_fault')
115 const outcomes = [...fill(998), aborted('a'), aborted('b')]
116 const window = judgeProbeWindow(outcomes, judgeOf(1 / 790, 100, 1_000))
117 assert.equal(window.kind, 'harness_fault')
118 if (window.kind === 'harness_fault') {
119 assert.equal(window.loss.count, 2)
120 assert.equal(window.loss.exceeded, true)
121 assert.match(window.reason, /exceeds probe.maxLossRate/)
122 }
123})
125test('losses that leave fewer responded probes than the minimum are insufficient samples, not a pass', () => {
126 const outcomes = [...fill(40), aborted('lost')]
127 const window = judgeProbeWindow(outcomes, judgeOf(0.5, 100, 1_000))
128 assert.equal(window.kind, 'insufficient_samples')
129 if (window.kind === 'insufficient_samples') {
130 assert.equal(window.loss.responded, 40)
131 assert.equal(window.loss.exceeded, false)
132 assert.match(window.reason, /40 responded probes < 100/)
133 }
134})
136test('a loss above the rate at 100% rolls back to previous and does not judge the step as a pass', async () => {
137 let rolled = 0
138 const splits: number[] = []
139 const result = await runRamp(policy(), {
140 now: () => 0,
141 sleep: async () => {},
142 dwell: async (_ms, percent) => ({
143 outcomes: percent === 100 ? [...fill(998), aborted('a'), aborted('b')] : fill(200),
144 }),
145 applySplit: async percent => {
146 splits.push(percent)
147 return null
148 },
149 observe: async (_from, _to, percent) => windowFor(percent),
150 rollback: async () => {
151 rolled += 1
152 return null
153 },
154 })
155 assert.deepEqual(splits, [75, 100])
156 assert.equal(rolled, 1)
157 assert.equal(result.state.kind, 'stopped')
158 if (result.state.kind === 'stopped' && result.state.fault === 'harness_fault') assert.equal(result.state.traffic, 'rolled_back')
159 const failed = result.steps[1]
160 if (!failed) throw new Error('missing 100% step')
161 assert.equal(failed.probeLoss.count, 2)
162 assert.equal(failed.outcome, 'harness_fault')
163})
165test('hold keeps the last fully passed split instead of rolling back to previous', async () => {
166 const splits: number[] = []
167 let rolled = 0
168 const result = await runRamp(policy({ onHarnessFault: 'hold', maxLossRate: 1 / 790, minSamples: 100 }), {
169 now: () => 0,
170 sleep: async () => {},
171 dwell: async (_ms, percent) => ({
172 outcomes: percent === 100 ? [...fill(998), aborted('a'), aborted('b')] : fill(200),
173 }),
174 applySplit: async percent => {
175 splits.push(percent)
176 return null
177 },
178 observe: async (_from, _to, percent) => windowFor(percent),
179 rollback: async () => {
180 rolled += 1
181 return null
182 },
183 })
184 assert.equal(rolled, 0)
185 assert.deepEqual(splits, [75, 100, 75])
186 if (result.state.kind === 'stopped' && result.state.fault === 'harness_fault') assert.equal(result.state.traffic, 'held')
187})
189// ply-web#145: the step is unjudged but its split already moved traffic, so onHarnessFault 'rollback' restores previous.
190test('a step under the minimum sample does not advance and rolls its traffic back', async () => {
191 let rolled = 0
192 const result = await runRamp(policy({ onHarnessFault: 'rollback', maxLossRate: 0.5, minSamples: 100 }), {
193 now: () => 0,
194 sleep: async () => {},
195 dwell: async () => ({ outcomes: [...fill(40), aborted('lost')] }),
196 applySplit: async () => null,
197 observe: async (_from, _to, percent) => windowFor(percent),
198 rollback: async () => {
199 rolled += 1
200 return null
201 },
202 })
203 assert.equal(rolled, 1)
204 assert.equal(result.state.kind, 'held')
205 if (result.state.kind === 'held') {
206 assert.equal(result.state.outcome, 'insufficient_samples')
207 assert.equal('traffic' in result.state ? result.state.traffic : null, 'rolled_back')
208 assert.match(result.state.reason, /rolled back to previous$/)
209 }
210 const recorded = result.steps[0]
211 if (!recorded) throw new Error('missing step')
212 assert.equal(recorded.outcome, 'insufficient_samples')
213})
Diagnostics4 git calls · 35 ms
CallStatusSizeTime035 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2002.5 KiB · hit29 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=29 git.response.bytes=2592 git.response.cache=HIT git.response.duration_ms=29 git.fetch.wants=51537cd9bd01ac35ce8f2bbccb338a4438e19eb7 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2592 git.pkt.payload.max=2571 git.sideband.band1.bytes=2570 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e32889cf9c4628
- Request
- 9510bf3e…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format