main
ply4 git calls38 ms45 KiB3 cachedprotocol v2
477 lines17 KiBRaw
1import assert from 'node:assert/strict'
2import { readFileSync } from 'node:fs'
3import { test } from 'node:test'
4import { none, some } from '../src/option.ts'
5import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
6import { classifyProbe, judgeProbeWindow, type ProbeJudge, type ProbeOutcome } from '../scripts/cd/probe.ts'
7import { runProbeSchedule, type ProbeClock } from '../scripts/cd/probe-schedule.ts'
8import { NO_STALLS } from '../scripts/cd/client-health.ts'
9import { runRamp, type Observation } from '../scripts/cd/ramp.ts'
10import type { VersionStats } from '../scripts/cd/telemetry.ts'
12const OLD = '4f3d7acf'
13const NEW = '19ee3cab'
15function policy(): RolloutPolicy {
16 const raw = JSON.parse(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')) as Record<string, unknown>
17 raw.ingestPollMs = 0
18 const parsed = parsePolicy(JSON.stringify(raw)).policy
19 // A lone 100 step is not a loadable policy (ramp-only, 2026-11-08); the engine runs one here to isolate a single window.
20 // Frozen test clocks need the fixed ingest sleep, and these windows were written against onHarnessFault rollback.
21 return { ...parsed, steps: [{ percent: 100, dwellMs: 40_000 }], probe: { ...parsed.probe, onHarnessFault: 'rollback' } }
24function judge(): ProbeJudge {
25 const rollout = policy()
26 return {
27 maxLossRate: rollout.probe.maxLossRate,
28 respondedMin: rollout.burn.minSamples,
29 minSamples: rollout.probe.minSamples,
30 stallLatencyMs: rollout.probe.stallLatencyMs,
31 fivexxHard: rollout.gates.fivexx === 'hard',
32 }
35function clock(): ProbeClock & { read(): number } {
36 let at = 0
37 return {
38 now: () => at,
39 sleep: async ms => {
40 at += ms
41 },
42 read: () => at,
43 }
46function answered(id: string, version: string, latencyMs: number, status = 200): ProbeOutcome {
47 return classifyProbe(id, { kind: 'response', status, ray: some('ray'), latencyMs, version: some(version) })
50function incident(): ProbeOutcome[] {
51 const outcomes: ProbeOutcome[] = []
52 for (let index = 0; index < 59; index += 1) {
53 outcomes.push(answered(`old-${index}`, OLD, 69_830))
54 outcomes.push(answered(`new-${index}`, NEW, 69_830))
55 }
56 for (let index = 0; index < 10; index += 1) outcomes.push({ kind: 'lost', id: `lost-${index}`, reason: 'network' })
57 return outcomes
60function clean(): ProbeOutcome[] {
61 const outcomes: ProbeOutcome[] = []
62 for (let index = 0; index < 100; index += 1) {
63 outcomes.push(answered(`old-${index}`, OLD, 50))
64 outcomes.push(answered(`new-${index}`, NEW, 50))
65 }
66 return outcomes
69function stats(requests: number): VersionStats {
70 return {
71 requests,
72 invocations: requests,
73 exceptions: 0,
74 errors: 0,
75 fivexx: 0,
76 errorLogs: 0,
77 wallMs: requests === 0 ? [] : [10, 10, 10, 10, 10],
78 cpuMs: requests === 0 ? [] : [10],
79 fetchMs: requests === 0 ? [] : [10],
80 v1: [],
81 internal: [],
82 }
85function at100(): Observation {
86 const next = stats(200)
87 return { kind: 'sample', next, previous: stats(0), aggregate: next, landed: next.invocations }
90test('a policy timeout is timed_out and a transport error stays network', () => {
91 const timed = classifyProbe('t', { kind: 'timeout' })
92 const network = classifyProbe('n', { kind: 'thrown', name: 'TypeError', message: 'fetch failed' })
93 assert.equal(timed.kind, 'lost')
94 if (timed.kind === 'lost') assert.equal(timed.reason, 'timed_out')
95 assert.equal(network.kind, 'lost')
96 if (network.kind === 'lost') assert.equal(network.reason, 'network')
97 const aborted = classifyProbe('a', { kind: 'thrown', name: 'AbortError', message: 'The operation was aborted' })
98 assert.equal(aborted.kind, 'lost')
99 if (aborted.kind === 'lost') assert.equal(aborted.reason, 'aborted')
100})
102test('at the in-flight bound the next tick is shed and the dwell stays on the wall clock', async () => {
103 const ticks = clock()
104 let stopped = 0
105 const outcomes = await runProbeSchedule({
106 clock: ticks,
107 dwellMs: 120,
108 periodMs: 40,
109 maxInFlight: 2,
110 batch: 2,
111 graceMs: 0,
112 health: NO_STALLS,
113 extension: { maxMs: 0, minReal: 0 },
114 issue: seq => ({ id: `probe-${seq}`, promise: new Promise<ProbeOutcome>(() => {}) }),
115 shed: seq => ({ kind: 'shed', id: `shed-${seq}` }),
116 stop: () => {
117 stopped += 1
118 },
119 })
120 assert.equal(ticks.read(), 120)
121 assert.equal(stopped, 1)
122 const shed = outcomes.filter(item => item.kind === 'shed')
123 const timed = outcomes.filter(item => item.kind === 'lost')
124 assert.equal(shed.length, 4)
125 assert.equal(timed.length, 2)
126 for (const item of timed) {
127 if (item.kind !== 'lost') throw new Error('expected a lost probe')
128 assert.equal(item.reason, 'timed_out')
129 }
130})
132test('a client stall does not stretch a 40s dwell and the unfinished probes are timed_out', async () => {
133 const ticks = clock()
134 const outcomes = await runProbeSchedule({
135 clock: ticks,
136 dwellMs: 40_000,
137 periodMs: 40,
138 maxInFlight: 4,
139 batch: 2,
140 graceMs: 0,
141 health: NO_STALLS,
142 extension: { maxMs: 0, minReal: 0 },
143 issue: seq => ({ id: `probe-${seq}`, promise: new Promise<ProbeOutcome>(() => {}) }),
144 shed: seq => ({ kind: 'shed', id: `shed-${seq}` }),
145 stop: () => {},
146 })
147 assert.equal(ticks.read(), 40_000)
148 const shed = outcomes.filter(item => item.kind === 'shed')
149 const timed = outcomes.filter(item => item.kind === 'lost')
150 assert.equal(timed.length, 4)
151 assert.ok(shed.length > 0)
152 const window = judgeProbeWindow(outcomes, judge())
153 assert.equal(window.kind, 'retry')
154 if (window.kind === 'retry') {
155 assert.equal(window.loss.serverErrors, 0)
156 assert.equal(window.loss.timedOut, 4)
157 assert.ok(window.loss.shed > 0)
158 }
159})
161test('a fast probe is issued on the period and is not shed', async () => {
162 const ticks = clock()
163 const outcomes = await runProbeSchedule({
164 clock: ticks,
165 dwellMs: 200,
166 periodMs: 40,
167 maxInFlight: 10,
168 batch: 2,
169 graceMs: 0,
170 health: NO_STALLS,
171 extension: { maxMs: 0, minReal: 0 },
172 issue: seq => ({
173 id: `probe-${seq}`,
174 promise: Promise.resolve({ kind: 'responded', id: `probe-${seq}`, status: 200, ray: none(), latencyMs: 40, version: none() }),
175 }),
176 shed: seq => ({ kind: 'shed', id: `shed-${seq}` }),
177 stop: () => {},
178 })
179 assert.equal(ticks.read(), 200)
180 assert.equal(outcomes.filter(item => item.kind === 'shed').length, 0)
181 assert.equal(outcomes.filter(item => item.kind === 'responded').length, 10)
182})
184test('a probe in flight at the deadline answers within the grace and is not lost', async () => {
185 // The last tick (t=80) is unanswered at the deadline (t=120); it answers once the grace wait has begun.
186 let at = 0
187 const late: (() => void)[] = []
188 const ticks: ProbeClock = {
189 now: () => at,
190 sleep: async ms => {
191 if (at >= 120) { for (const release of late.splice(0)) release(); await Promise.resolve(); return }
192 at += ms
193 },
194 }
195 let stopped = 0
196 const outcomes = await runProbeSchedule({
197 clock: ticks,
198 dwellMs: 120,
199 periodMs: 40,
200 maxInFlight: 10,
201 batch: 2,
202 graceMs: 2000,
203 health: NO_STALLS,
204 extension: { maxMs: 0, minReal: 0 },
205 issue: seq => {
206 const answer: ProbeOutcome = { kind: 'responded', id: `probe-${seq}`, status: 200, ray: none(), latencyMs: 40, version: none() }
207 if (seq <= 4) return { id: `probe-${seq}`, promise: Promise.resolve(answer) }
208 return { id: `probe-${seq}`, promise: new Promise<ProbeOutcome>(resolve => { late.push(() => resolve(answer)) }) }
209 },
210 shed: seq => ({ kind: 'shed', id: `shed-${seq}` }),
211 stop: () => {
212 stopped += 1
213 },
214 })
215 assert.equal(stopped, 1)
216 assert.equal(outcomes.filter(item => item.kind === 'lost').length, 0)
217 assert.equal(outcomes.filter(item => item.kind === 'responded').length, 6)
218})
220test('a probe still hung after the grace is timed_out and the dwell ends at deadline + grace', async () => {
221 const ticks = clock()
222 const outcomes = await runProbeSchedule({
223 clock: ticks,
224 dwellMs: 120,
225 periodMs: 40,
226 maxInFlight: 10,
227 batch: 2,
228 graceMs: 2000,
229 health: NO_STALLS,
230 extension: { maxMs: 0, minReal: 0 },
231 issue: seq => ({ id: `probe-${seq}`, promise: new Promise<ProbeOutcome>(() => {}) }),
232 shed: seq => ({ kind: 'shed', id: `shed-${seq}` }),
233 stop: () => {},
234 })
235 assert.equal(ticks.read(), 2120)
236 assert.equal(outcomes.filter(item => item.kind === 'lost').length, 6)
237})
239test('10/128 above the loss rate does not fail a window under probe.minSamples', () => {
240 const rollout = policy()
241 const outcomes: ProbeOutcome[] = []
242 for (let index = 0; index < 118; index += 1) outcomes.push(answered(`ok-${index}`, OLD, 50))
243 for (let index = 0; index < 10; index += 1) outcomes.push({ kind: 'lost', id: `lost-${index}`, reason: 'network' })
244 const window = judgeProbeWindow(outcomes, judge())
245 assert.equal(window.loss.total, 128)
246 assert.equal(window.loss.count, 10)
247 assert.equal(window.loss.exceeded, true)
248 assert.ok(window.loss.total < rollout.probe.minSamples)
249 assert.equal(window.kind, 'ok')
250})
252test('the same stall on both versions re-runs the dwell and does not roll back', async () => {
253 const rollout = policy()
254 let dwells = 0
255 let rolled = 0
256 const waits: number[] = []
257 const result = await runRamp(rollout, {
258 now: () => 0,
259 sleep: async ms => {
260 waits.push(ms)
261 },
262 dwell: async () => {
263 dwells += 1
264 return { outcomes: dwells === 1 ? incident() : clean() }
265 },
266 applySplit: async () => null,
267 observe: async () => at100(),
268 rollback: async () => {
269 rolled += 1
270 return null
271 },
272 })
273 assert.equal(dwells, rollout.probe.windowAttempts)
274 assert.equal(rolled, 0)
275 assert.equal(result.state.kind, 'complete')
276 // the policy's warm-up (the split reaching every edge) comes first, outside the judged window
277 assert.deepEqual(waits, [...(rollout.warmupMs > 0 ? [rollout.warmupMs] : []), rollout.probe.windowBackoffMs, rollout.ingestWaitMs])
278})
280// ply-web#145: insufficient_samples is unjudged, so the step's traffic is settled through onHarnessFault (this file's policy: rollback).
281test('a stall that is still there after the bounded re-runs is unjudged and rolls back to previous', async () => {
282 const rollout = policy()
283 let dwells = 0
284 let rolled = 0
285 const result = await runRamp(rollout, {
286 now: () => 0,
287 sleep: async () => {},
288 dwell: async () => {
289 dwells += 1
290 return { outcomes: incident() }
291 },
292 applySplit: async () => null,
293 observe: async () => at100(),
294 rollback: async () => {
295 rolled += 1
296 return null
297 },
298 })
299 assert.equal(dwells, rollout.probe.windowAttempts)
300 assert.equal(rollout.probe.onHarnessFault, 'rollback')
301 assert.equal(rolled, 1)
302 assert.equal(result.state.kind, 'held')
303 if (result.state.kind === 'held') {
304 assert.equal(result.state.outcome, 'insufficient_samples')
305 assert.equal('traffic' in result.state ? result.state.traffic : null, 'rolled_back')
306 assert.match(result.state.reason, /harness\/network stall/)
307 assert.doesNotMatch(result.state.reason, /not rolling back|prod untouched/)
308 assert.match(result.state.reason, /0 server errors/)
309 assert.match(result.state.reason, /rolled back to previous$/)
310 }
311})
313test('a 5xx on a responded probe still rolls back inside a stall window', async () => {
314 const stalled = incident()
315 stalled[0] = answered('old-500', OLD, 69_830, 500)
316 let dwells = 0
317 let rolled = 0
318 const result = await runRamp(policy(), {
319 now: () => 0,
320 sleep: async () => {},
321 dwell: async () => {
322 dwells += 1
323 return { outcomes: stalled }
324 },
325 applySplit: async () => null,
326 observe: async () => at100(),
327 rollback: async () => {
328 rolled += 1
329 return null
330 },
331 })
332 assert.equal(dwells, 1)
333 assert.equal(rolled, 1)
334 assert.equal(result.state.kind, 'rolled-back')
335 if (result.state.kind === 'rolled-back') assert.match(result.state.reason, /1 5xx on responded probes/)
336})
338test('a stall on only one version still fails the loss-rate gate', () => {
339 const outcomes: ProbeOutcome[] = []
340 for (let index = 0; index < 500; index += 1) {
341 outcomes.push(answered(`old-${index}`, OLD, 50))
342 outcomes.push(answered(`new-${index}`, NEW, 69_830))
343 }
344 for (let index = 0; index < 10; index += 1) outcomes.push({ kind: 'lost', id: `lost-${index}`, reason: 'network' })
345 const window = judgeProbeWindow(outcomes, judge())
346 assert.equal(window.kind, 'harness_fault')
347})
349// ---- ply-web#145 review round 2: settle paths pinned (hand-derived expectations from the issue's acceptance criteria) ----
351/** A sample whose routed share matches `percent`: 200 invocations split by the step's percentage. */
352function atPercent(percent: number): Observation {
353 const next = stats(2 * percent)
354 const previous = stats(200 - 2 * percent)
355 return { kind: 'sample', next, previous, aggregate: stats(200), landed: 200 }
358function unhealthyWindow(): ProbeOutcome[] {
359 const outcomes: ProbeOutcome[] = []
360 for (let index = 0; index < 53; index += 1) outcomes.push(answered(`ok-${index}`, NEW, 50))
361 outcomes.push({ kind: 'stalled', id: 'client-stall 0-53733 53733ms' })
362 return outcomes
365/** 75% passes on a clean window, then the 100% step gets `bad` on every attempt; onHarnessFault is `hold`. */
366async function holdAfterPass(bad: () => ProbeOutcome[], rollback: () => Promise<null | string> = async () => null) {
367 const base = policy()
368 const rollout: RolloutPolicy = {
369 ...base,
370 steps: [{ percent: 75, dwellMs: 40_000 }, { percent: 100, dwellMs: 40_000 }],
371 probe: { ...base.probe, onHarnessFault: 'hold' },
372 }
373 const counts = { rolled: 0, splits: [] as number[] }
374 let current = 0
375 const result = await runRamp(rollout, {
376 now: () => 0,
377 sleep: async () => {},
378 dwell: async () => ({ outcomes: current === 75 ? clean() : bad() }),
379 applySplit: async percent => {
380 counts.splits.push(percent)
381 current = percent
382 return null
383 },
384 observe: async (_from, _to, percent) => atPercent(percent),
385 rollback: async () => {
386 counts.rolled += 1
387 return rollback()
388 },
389 })
390 return { result, counts }
393for (const [name, bad, outcome] of [
394 ['harness_unhealthy', unhealthyWindow, 'harness_unhealthy'],
395 ['insufficient_samples', incident, 'insufficient_samples'],
396] as const) {
397 test(`hold: ${name} at 100% after 75% passed writes the split back to 75% and does not roll back`, async () => {
398 const { result, counts } = await holdAfterPass(bad)
399 assert.deepEqual(counts.splits, [75, 100, 75])
400 assert.equal(counts.rolled, 0)
401 assert.equal(result.rollbackError, null)
402 assert.equal(result.state.kind, 'held')
403 if (result.state.kind === 'held') {
404 assert.equal(result.state.outcome, outcome)
405 assert.equal('traffic' in result.state ? result.state.traffic : null, 'held')
406 assert.match(result.state.reason, /; held at 75%$/)
407 assert.doesNotMatch(result.state.reason, /prod untouched/)
408 }
409 })
412test('a failed rollback of an unjudged 100% step says traffic is still at 100%, not that it rolled back', async () => {
413 const rollout = { ...policy(), steps: [{ percent: 100, dwellMs: 40_000 }] }
414 assert.equal(rollout.probe.onHarnessFault, 'rollback')
415 const result = await runRamp(rollout, {
416 now: () => 0,
417 sleep: async () => {},
418 dwell: async () => ({ outcomes: unhealthyWindow() }),
419 applySplit: async () => null,
420 observe: async () => at100(),
421 rollback: async () => 'boom 503',
422 })
423 assert.equal(result.rollbackError, 'boom 503')
424 assert.equal(result.state.kind, 'held')
425 if (result.state.kind === 'held') {
426 assert.equal('traffic' in result.state ? result.state.traffic : null, 'settle_failed')
427 assert.doesNotMatch(result.state.reason, /rolled back to previous/)
428 assert.match(result.state.reason, /; rollback to previous failed: boom 503; still at 100%$/)
429 }
430})
432test('a failed hold write says the split is still at the stopped step', async () => {
433 const base = policy()
434 const rollout: RolloutPolicy = {
435 ...base,
436 steps: [{ percent: 75, dwellMs: 40_000 }, { percent: 100, dwellMs: 40_000 }],
437 probe: { ...base.probe, onHarnessFault: 'hold' },
438 }
439 let current = 0
440 let writes = 0
441 const result = await runRamp(rollout, {
442 now: () => 0,
443 sleep: async () => {},
444 dwell: async () => ({ outcomes: current === 75 ? clean() : unhealthyWindow() }),
445 applySplit: async percent => {
446 writes += 1
447 if (writes === 3) return 'split 500'
448 current = percent
449 return null
450 },
451 observe: async (_from, _to, percent) => atPercent(percent),
452 rollback: async () => null,
453 })
454 assert.equal(result.rollbackError, 'split 500')
455 assert.equal(result.state.kind === 'held' && 'traffic' in result.state ? result.state.traffic : null, 'settle_failed')
456 if (result.state.kind === 'held') assert.match(result.state.reason, /; hold at 75% failed: split 500; still at 100%$/)
457})
459// the ramp's at-0% branch: nothing to settle, and the reason says so
460test('an unjudged step at 0% holds at 0% without a rollback', async () => {
461 const rollout = { ...policy(), steps: [{ percent: 0, dwellMs: 40_000 }] }
462 let rolled = 0
463 const result = await runRamp(rollout, {
464 now: () => 0,
465 sleep: async () => {},
466 dwell: async () => ({ outcomes: unhealthyWindow() }),
467 applySplit: async () => null,
468 observe: async () => atPercent(0),
469 rollback: async () => {
470 rolled += 1
471 return null
472 },
473 })
474 assert.equal(rolled, 0)
475 assert.equal(result.state.kind === 'held' && 'traffic' in result.state ? result.state.traffic : null, 'held')
476 if (result.state.kind === 'held') assert.match(result.state.reason, /; held at 0%$/)
477})
Diagnostics4 git calls · 38 ms
CallStatusSizeTime038 ms
discover200152 B4 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=4
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=4
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=4
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2004.1 KiB · hit32 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=31
git.response.bytes=4159
git.response.cache=HIT
git.response.duration_ms=32
git.fetch.wants=c0075c4bcd190a6cd4b31f83838f45220cb225f2
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4159
git.pkt.payload.max=4138
git.sideband.band1.bytes=4137
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e30b903837ee28
Request
991e1f95…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format