main
ply4 git calls186 ms44 KiB2 cachedprotocol v2
304 lines12 KiBRaw
1import assert from 'node:assert/strict'
2import { readFileSync } from 'node:fs'
3import { test } from 'node:test'
4import { none } from '../src/option.ts'
5import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
6import type { ProbeOutcome } from '../scripts/cd/probe.ts'
7import { applyRamp, decideRampStep, progressLine, rampStart, runRamp, type Observation } from '../scripts/cd/ramp.ts'
8import type { VersionStats } from '../scripts/cd/telemetry.ts'
9import { routedShare } from '../scripts/cd/slo.ts'
11function policy(steps?: { percent: number; dwellMs: number }[], warmupMs = 0): RolloutPolicy {
12 const raw = JSON.parse(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')) as {
13 steps: { percent: number; dwellMs: number }[]
14 warmupMs: number
15 }
16 raw.warmupMs = warmupMs
17 const parsed = parsePolicy(JSON.stringify(raw)).policy
18 // A one-step schedule is not a loadable policy (ramp-only, 2026-11-08); these engine tests drive runRamp with one directly.
19 // The engine tests below exercise onHarnessFault rollback, the mode the file shipped before 2026-11-08.
20 return { ...parsed, steps: steps ? steps : parsed.steps, probe: { ...parsed.probe, onHarnessFault: 'rollback' } }
23function stats(over: Partial<VersionStats> = {}): VersionStats {
24 const row: VersionStats = {
25 requests: 20,
26 invocations: 0,
27 exceptions: 0,
28 errors: 0,
29 fivexx: 0,
30 errorLogs: 0,
31 wallMs: [80, 90, 100, 110, 120],
32 cpuMs: [10, 12, 14],
33 fetchMs: [40, 50, 60],
34 v1: [],
35 internal: [],
36 ...over,
37 }
38 return { ...row, invocations: over.invocations ?? row.requests }
41function sample(next: Partial<VersionStats> = {}, previous: Partial<VersionStats> = {}): Observation {
42 const newer = stats(next)
43 const older = stats(previous)
44 return {
45 kind: 'sample',
46 next: newer,
47 previous: older,
48 aggregate: stats({
49 requests: newer.requests + older.requests,
50 invocations: newer.invocations + older.invocations,
51 exceptions: newer.exceptions + older.exceptions,
52 errors: newer.errors + older.errors,
53 fivexx: newer.fivexx + older.fivexx,
54 errorLogs: newer.errorLogs + older.errorLogs,
55 wallMs: [...newer.wallMs, ...older.wallMs],
56 cpuMs: [...newer.cpuMs, ...older.cpuMs],
57 fetchMs: [...newer.fetchMs, ...older.fetchMs],
58 v1: [...newer.v1, ...older.v1],
59 }),
60 landed: newer.invocations + older.invocations,
61 }
64function atPercent(percent: number): Observation {
65 if (percent >= 100) {
66 return sample(
67 { requests: 200, invocations: 200, wallMs: [10, 10, 10, 10, 10] },
68 { requests: 0, invocations: 0, wallMs: [], cpuMs: [], fetchMs: [] },
69 )
70 }
71 return sample(
72 { requests: 200, invocations: 200, wallMs: [10, 10, 10, 10, 10] },
73 { requests: 200, invocations: 200, wallMs: [10, 10, 10, 10, 10] },
74 )
77const twoSteps = policy([{ percent: 50, dwellMs: 1 }, { percent: 100, dwellMs: 1 }])
79test('a short clean sample is insufficient evidence and holds', () => {
80 const decision = decideRampStep(
81 sample({ requests: 2, invocations: 2, wallMs: [80, 81] }, { requests: 3, invocations: 3, wallMs: [70, 71, 72] }),
82 twoSteps,
83 50,
84 )
85 assert.equal(decision.outcome, 'insufficient_data')
86 assert.equal(applyRamp(rampStart(), decision, 5).kind, 'held')
87 const ready = decideRampStep(atPercent(50), twoSteps, 50)
88 assert.equal(ready.outcome, 'advance')
89 assert.deepEqual(applyRamp(rampStart(), ready, 5), { kind: 'ramping', step: 1 })
90 assert.equal(applyRamp({ kind: 'ramping', step: 4 }, ready, 5).kind, 'complete')
91})
93test('unavailable telemetry above 0% is a harness fault and rolls back to previous', async () => {
94 const decision = decideRampStep({ kind: 'unavailable', reason: 'telemetry unavailable: cf o11y down' }, twoSteps, 50)
95 assert.equal(decision.outcome, 'harness_fault')
96 assert.match(decision.reason, /telemetry unavailable/)
97 assert.equal(decision.rules[0]?.name, 'telemetry_required')
98 let rolled = 0
99 const result = await runRamp(twoSteps, {
100 now: () => 0,
101 sleep: async () => {},
102 applySplit: async () => null,
103 observe: async () => ({ kind: 'unavailable', reason: 'telemetry unavailable: cf o11y down' }),
104 rollback: async () => {
105 rolled += 1
106 return null
107 },
108 })
109 assert.equal(result.state.kind, 'stopped')
110 if (result.state.kind === 'stopped' && result.state.fault === 'harness_fault') assert.equal(result.state.traffic, 'rolled_back')
111 assert.equal(rolled, 1)
112 assert.match(result.lines[0] ?? '', /decision=harness_fault/)
113})
115test('one exception on the new version rolls back before the sample minimum', () => {
116 const decision = decideRampStep(sample({ requests: 1, invocations: 1, exceptions: 1, errors: 1, wallMs: [40] }, { requests: 40, invocations: 40 }), twoSteps, 50)
117 assert.equal(decision.outcome, 'rollback')
118 assert.equal(decision.reason, '1 exception on the new version')
119})
121test('a large wall shift stays an advance under the checked-in policy', () => {
122 const over = decideRampStep(sample(
123 { requests: 200, invocations: 200, wallMs: [210, 210, 210, 210, 210, 210] },
124 { requests: 200, invocations: 200, wallMs: [80, 80, 80, 80, 80, 80] },
125 ), twoSteps, 50)
126 assert.equal(over.outcome, 'advance')
127})
129test('a routed share outside the band rolls back', () => {
130 const decision = decideRampStep(sample(
131 { requests: 1000, invocations: 1000 },
132 { requests: 1000, invocations: 1000 },
133 ), twoSteps, 10)
134 assert.equal(decision.outcome, 'rollback')
135 assert.equal(decision.rules[0]?.name, 'routed_share')
136})
138test('applyRamp ignores decisions after rollback', () => {
139 const rolled = { kind: 'rolled-back' as const, reason: 'breach' }
140 assert.equal(applyRamp(rolled, decideRampStep(atPercent(50), twoSteps, 50), 5), rolled)
141})
143test('progress line labels latency informational and names a carried baseline', () => {
144 const share = routedShare(200, 400, 50, 3)
145 const advance = {
146 outcome: 'advance' as const,
147 reason: 'within burn 14.4',
148 rules: [{ name: 'fast_burn' as const, value: 14.4 }] as [{ name: string; value: number }],
149 burns: { availability: null, read_latency: null, write_latency: null, correctness: null },
150 }
151 const line = progressLine({
152 step: 1,
153 steps: 5,
154 percent: 50,
155 next: stats({ requests: 200, invocations: 200, wallMs: [88] }),
156 previous: stats({ requests: 200, invocations: 200, wallMs: [74] }),
157 carriedPrevious: null,
158 share,
159 decision: advance,
160 latency: 'informational',
161 probe: { kind: 'none' },
162 sample: { kind: 'probes', rays: ['a585f4159ae599a2-IAD'] },
163 })
164 assert.match(line, /^ramp step=1\/5 percent=50 previous=50 /)
165 assert.match(line, /routed_share=50.00 target=50 tolerance=42.50..57.50 agg=400/)
166 assert.match(line, /latency=informational latency_sample=probes/)
167 assert.match(line, /decision=advance/)
168 const carried = progressLine({
169 step: 5,
170 steps: 5,
171 percent: 100,
172 next: stats({ requests: 120, invocations: 120, wallMs: [114, 114, 114, 114, 114] }),
173 previous: stats({ requests: 0, invocations: 0, wallMs: [], cpuMs: [], fetchMs: [] }),
174 carriedPrevious: stats({ requests: 28, invocations: 28, wallMs: [84, 84, 84, 84, 84] }),
175 share: routedShare(120, 120, 100, 3),
176 decision: advance,
177 latency: 'informational',
178 probe: { kind: 'none' },
179 sample: { kind: 'probes', rays: ['a585f4159ae599a2-IAD'] },
180 })
181 assert.equal(
182 carried,
183 'ramp step=5/5 percent=100 previous=0 samples=120/0 error_rate=0.0000/- routed_share=100.00 target=100 tolerance=100.00..100.00 agg=120 p99_wall_ms=114/84 p99_cpu_ms=12/12 latency=informational latency_sample=probes decision=advance previous_sample=carried',
184 )
185})
187test('at 100% the old side is the last wall sample that met the quantile floor', async () => {
188 const enough = stats({ requests: 100, invocations: 100, wallMs: [84, 84, 84, 84, 84] })
189 const quiet = stats({ requests: 0, invocations: 0, wallMs: [], cpuMs: [], fetchMs: [] })
190 let step = 0
191 const result = await runRamp(twoSteps, {
192 now: () => 0,
193 sleep: async () => {},
194 applySplit: async () => null,
195 observe: async () => {
196 step += 1
197 if (step === 1) return sample({ requests: 100, invocations: 100, wallMs: [80, 80, 80, 80, 80] }, enough)
198 return sample({ requests: 100, invocations: 100, wallMs: [90, 90, 90, 90, 90] }, quiet)
199 },
200 rollback: async () => null,
201 })
202 assert.equal(result.state.kind, 'complete')
203 assert.match(result.lines[1] ?? '', /previous_sample=carried/)
204 assert.match(result.lines[1] ?? '', /samples=100\/0/)
205 assert.match(result.lines[1] ?? '', /p99_wall_ms=90\/84/)
206})
208test('a harness fault during the dwell rolls back and records the loss', async () => {
209 let rolled = 0
210 let observed = 0
211 const result = await runRamp(policy([{ percent: 100, dwellMs: 1 }]), {
212 now: () => 0,
213 sleep: async () => {},
214 dwell: async () => {
215 const outcomes: ProbeOutcome[] = []
216 for (let index = 0; index < 998; index += 1) {
217 outcomes.push({ kind: 'responded', id: `ok-${index}`, status: 200, ray: none(), latencyMs: 50, version: none() })
218 }
219 outcomes.push({ kind: 'lost', id: '1 GET https://ply.ivar.workers.dev/', reason: 'aborted' })
220 outcomes.push({ kind: 'lost', id: '2 GET https://ply.ivar.workers.dev/', reason: 'aborted' })
221 return { outcomes }
222 },
223 applySplit: async () => null,
224 observe: async () => {
225 observed += 1
226 return sample()
227 },
228 rollback: async () => {
229 rolled += 1
230 return null
231 },
232 })
233 assert.equal(result.state.kind, 'stopped')
234 if (result.state.kind === 'stopped' && result.state.fault === 'harness_fault') {
235 assert.equal(result.state.traffic, 'rolled_back')
236 assert.match(result.state.reason, /probe loss 2\/1000/)
237 }
238 assert.equal(rolled, 1)
239 assert.equal(observed, 0)
240 const recorded = result.steps[0]
241 if (!recorded) throw new Error('missing probe step')
242 assert.equal(recorded.probeLoss.ids[0], '1 GET https://ply.ivar.workers.dev/')
243 const recordedLine = result.lines[0]
244 if (!recordedLine) throw new Error('missing probe line')
245 assert.match(recordedLine, /probe_loss=2\/1000/)
246 assert.match(recordedLine, /timed_out=0/)
247 assert.match(recordedLine, /shed=0/)
248})
250test('a state conflict stops and does not roll back', async () => {
251 let rolled = 0
252 const result = await runRamp(twoSteps, {
253 now: () => 0,
254 sleep: async () => {},
255 applySplit: async () => ({ kind: 'state_conflict', reason: 'deployment does not match the expected prior state' }),
256 observe: async () => sample(),
257 rollback: async () => {
258 rolled += 1
259 return null
260 },
261 })
262 assert.deepEqual(result.state, {
263 kind: 'stopped',
264 reason: 'deployment does not match the expected prior state',
265 fault: 'state_conflict',
266 })
267 assert.equal(rolled, 0)
268})
270test('the ramp waits out the policy ingest lag, and a warm-up, before it reads the step', async () => {
271 const slept: number[] = []
272 const result = await runRamp(policy([{ percent: 100, dwellMs: 40 }]), {
273 now: () => 0,
274 sleep: async ms => { slept.push(ms) },
275 applySplit: async () => null,
276 observe: async () => atPercent(100),
277 rollback: async () => null,
278 })
279 assert.equal(result.state.kind, 'complete')
280 assert.deepEqual(slept, [40, 20_000])
281 const warmed: number[] = []
282 await runRamp(policy([{ percent: 100, dwellMs: 40 }], 5_000), {
283 now: () => 0,
284 sleep: async ms => { warmed.push(ms) },
285 applySplit: async () => null,
286 observe: async () => atPercent(100),
287 rollback: async () => null,
288 })
289 assert.deepEqual(warmed, [5_000, 40, 20_000])
290})
292test('a decision callback without an observation ref throws', async () => {
293 await assert.rejects(runRamp(
294 policy([{ percent: 100, dwellMs: 1 }]),
295 {
296 now: () => 0,
297 sleep: async () => {},
298 applySplit: async () => null,
299 observe: async () => sample({ requests: 200, invocations: 200 }),
300 onDecision: async () => {},
301 rollback: async () => null,
302 },
303 ))
304})
Diagnostics4 git calls · 186 ms
CallStatusSizeTime0186 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2003.2 KiB181 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=181
git.response.bytes=3290
git.response.cache=MISS
git.response.duration_ms=181
git.fetch.wants=3d153796c1ede2cedf7a3e4a4bedc3cc83389ae5
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3290
git.pkt.payload.max=3269
git.sideband.band1.bytes=3268
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e30f5208c38a31
Request
fa69d6d8…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format