main
ply4 git calls35 ms43 KiB3 cachedprotocol v2
213 lines8.9 KiBRaw
1import assert from 'node:assert/strict'
2import { readFileSync } from 'node:fs'
3import { test } from 'node:test'
4import { none, some } from '../src/option.ts'
5import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
6import { accountProbeLoss, classifyProbe, judgeProbeWindow, type ProbeJudge, type ProbeOutcome } from '../scripts/cd/probe.ts'
7import { runRamp, type Observation } from '../scripts/cd/ramp.ts'
8import type { VersionStats } from '../scripts/cd/telemetry.ts'
10function policy(over: { readonly onHarnessFault: 'rollback' | 'hold'; readonly maxLossRate: number; readonly minSamples: number } | null = null): RolloutPolicy {
11 const raw = JSON.parse(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')) as {
12 steps: { percent: number; dwellMs: number }[]
13 probe: { maxLossRate: number; onHarnessFault: string }
14 burn: { minSamples: number }
15 ingestPollMs: number
16 }
17 raw.steps = [{ percent: 75, dwellMs: 1 }, { percent: 100, dwellMs: 1 }]
18 // These tests drive the probe-loss gate's traffic modes with a frozen clock: the fixed ingest sleep (a polled wait
19 // needs a clock that moves), and onHarnessFault rollback unless a test picks another (the file is telemetry since
20 // 2026-11-08; test/cd-ramp-only.test.ts covers that mode).
21 raw.ingestPollMs = 0
22 raw.probe.onHarnessFault = 'rollback'
23 if (over) {
24 raw.probe.maxLossRate = over.maxLossRate
25 raw.probe.onHarnessFault = over.onHarnessFault
26 raw.burn.minSamples = over.minSamples
27 }
28 return parsePolicy(JSON.stringify(raw)).policy
31function judgeOf(maxLossRate: number, respondedMin: number, minSamples: number): ProbeJudge {
32 return { maxLossRate, respondedMin, minSamples, stallLatencyMs: 1_000, fivexxHard: true }
35function responded(id: string, status = 200): ProbeOutcome {
36 return classifyProbe(id, { kind: 'response', status, ray: some('ray'), latencyMs: 50, version: none() })
39function aborted(id: string): ProbeOutcome {
40 return classifyProbe(id, { kind: 'thrown', name: 'AbortError', message: 'The operation was aborted' })
43function fill(count: number, status = 200): ProbeOutcome[] {
44 const outcomes: ProbeOutcome[] = []
45 for (let index = 0; index < count; index++) outcomes.push(responded(`ok-${index}`, status))
46 return outcomes
49function stats(requests: number): VersionStats {
50 return {
51 requests,
52 invocations: requests,
53 exceptions: 0,
54 errors: 0,
55 fivexx: 0,
56 errorLogs: 0,
57 wallMs: [10, 10, 10, 10, 10],
58 cpuMs: [10],
59 fetchMs: [10],
60 v1: [],
61 internal: [],
62 }
65function windowFor(percent: number): Observation {
66 if (percent >= 100) {
67 const next = stats(200)
68 const previous = stats(0)
69 return { kind: 'sample', next, previous: { ...previous, wallMs: [], cpuMs: [], fetchMs: [] }, aggregate: next, landed: next.invocations }
70 }
71 const next = stats(150)
72 const previous = stats(50)
73 const aggregate = stats(200)
74 return { kind: 'sample', next, previous, aggregate, landed: aggregate.invocations }
77test('a client abort with no status and no ray is a loss, and a 500 is a responded server error', () => {
78 const lost = aborted('1 GET https://ply.ivar.workers.dev/')
79 assert.equal(lost.kind, 'lost')
80 if (lost.kind === 'lost') assert.equal(lost.reason, 'aborted')
81 const timed = classifyProbe('timed', { kind: 'timeout' })
82 assert.deepEqual(timed, { kind: 'lost', id: 'timed', reason: 'timed_out' })
83 const named = classifyProbe('named', { kind: 'thrown', name: 'TimeoutError', message: 'The operation was aborted due to timeout' })
84 assert.deepEqual(named, { kind: 'lost', id: 'named', reason: 'timed_out' })
85 const network = classifyProbe('net', { kind: 'thrown', name: 'TypeError', message: 'fetch failed' })
86 assert.deepEqual(network, { kind: 'lost', id: 'net', reason: 'network' })
87 const empty = classifyProbe('empty', { kind: 'response', status: 0, ray: some('ray'), latencyMs: 50, version: none() })
88 assert.equal(empty.kind, 'lost')
89 const server = classifyProbe('srv', { kind: 'response', status: 500, ray: some('ray'), latencyMs: 50, version: none() })
90 assert.equal(server.kind, 'responded')
91 const loss = accountProbeLoss([lost, server, ...fill(1)], 1 / 790)
92 assert.equal(loss.count, 1)
93 assert.equal(loss.responded, 2)
94 assert.equal(loss.serverErrors, 1)
95 assert.deepEqual(loss.ids, ['1 GET https://ply.ivar.workers.dev/'])
96 assert.equal(loss.exceeded, true)
97})
99test('the measured 1/790 abort is under the checked-in rate and is not a server error', () => {
100 const outcomes = [...fill(789), aborted('790 GET https://ply.ivar.workers.dev/')]
101 const loss = accountProbeLoss(outcomes, 1 / 790)
102 assert.equal(loss.total, 790)
103 assert.equal(loss.count, 1)
104 assert.equal(loss.rate, 1 / 790)
105 assert.equal(loss.exceeded, false)
106 assert.equal(loss.responded, 789)
107 assert.equal(loss.serverErrors, 0)
108 const window = judgeProbeWindow(outcomes, judgeOf(1 / 790, 100, 1_000))
109 assert.equal(window.kind, 'ok')
110})
112test('one more loss than the measured rate is a harness fault once the denominator is large enough', () => {
113 const under = judgeProbeWindow([...fill(788), aborted('a'), aborted('b')], judgeOf(1 / 790, 100, 1_000))
114 assert.notEqual(under.kind, 'harness_fault')
115 const outcomes = [...fill(998), aborted('a'), aborted('b')]
116 const window = judgeProbeWindow(outcomes, judgeOf(1 / 790, 100, 1_000))
117 assert.equal(window.kind, 'harness_fault')
118 if (window.kind === 'harness_fault') {
119 assert.equal(window.loss.count, 2)
120 assert.equal(window.loss.exceeded, true)
121 assert.match(window.reason, /exceeds probe.maxLossRate/)
122 }
123})
125test('losses that leave fewer responded probes than the minimum are insufficient samples, not a pass', () => {
126 const outcomes = [...fill(40), aborted('lost')]
127 const window = judgeProbeWindow(outcomes, judgeOf(0.5, 100, 1_000))
128 assert.equal(window.kind, 'insufficient_samples')
129 if (window.kind === 'insufficient_samples') {
130 assert.equal(window.loss.responded, 40)
131 assert.equal(window.loss.exceeded, false)
132 assert.match(window.reason, /40 responded probes < 100/)
133 }
134})
136test('a loss above the rate at 100% rolls back to previous and does not judge the step as a pass', async () => {
137 let rolled = 0
138 const splits: number[] = []
139 const result = await runRamp(policy(), {
140 now: () => 0,
141 sleep: async () => {},
142 dwell: async (_ms, percent) => ({
143 outcomes: percent === 100 ? [...fill(998), aborted('a'), aborted('b')] : fill(200),
144 }),
145 applySplit: async percent => {
146 splits.push(percent)
147 return null
148 },
149 observe: async (_from, _to, percent) => windowFor(percent),
150 rollback: async () => {
151 rolled += 1
152 return null
153 },
154 })
155 assert.deepEqual(splits, [75, 100])
156 assert.equal(rolled, 1)
157 assert.equal(result.state.kind, 'stopped')
158 if (result.state.kind === 'stopped' && result.state.fault === 'harness_fault') assert.equal(result.state.traffic, 'rolled_back')
159 const failed = result.steps[1]
160 if (!failed) throw new Error('missing 100% step')
161 assert.equal(failed.probeLoss.count, 2)
162 assert.equal(failed.outcome, 'harness_fault')
163})
165test('hold keeps the last fully passed split instead of rolling back to previous', async () => {
166 const splits: number[] = []
167 let rolled = 0
168 const result = await runRamp(policy({ onHarnessFault: 'hold', maxLossRate: 1 / 790, minSamples: 100 }), {
169 now: () => 0,
170 sleep: async () => {},
171 dwell: async (_ms, percent) => ({
172 outcomes: percent === 100 ? [...fill(998), aborted('a'), aborted('b')] : fill(200),
173 }),
174 applySplit: async percent => {
175 splits.push(percent)
176 return null
177 },
178 observe: async (_from, _to, percent) => windowFor(percent),
179 rollback: async () => {
180 rolled += 1
181 return null
182 },
183 })
184 assert.equal(rolled, 0)
185 assert.deepEqual(splits, [75, 100, 75])
186 if (result.state.kind === 'stopped' && result.state.fault === 'harness_fault') assert.equal(result.state.traffic, 'held')
187})
189// ply-web#145: the step is unjudged but its split already moved traffic, so onHarnessFault 'rollback' restores previous.
190test('a step under the minimum sample does not advance and rolls its traffic back', async () => {
191 let rolled = 0
192 const result = await runRamp(policy({ onHarnessFault: 'rollback', maxLossRate: 0.5, minSamples: 100 }), {
193 now: () => 0,
194 sleep: async () => {},
195 dwell: async () => ({ outcomes: [...fill(40), aborted('lost')] }),
196 applySplit: async () => null,
197 observe: async (_from, _to, percent) => windowFor(percent),
198 rollback: async () => {
199 rolled += 1
200 return null
201 },
202 })
203 assert.equal(rolled, 1)
204 assert.equal(result.state.kind, 'held')
205 if (result.state.kind === 'held') {
206 assert.equal(result.state.outcome, 'insufficient_samples')
207 assert.equal('traffic' in result.state ? result.state.traffic : null, 'rolled_back')
208 assert.match(result.state.reason, /rolled back to previous$/)
209 }
210 const recorded = result.steps[0]
211 if (!recorded) throw new Error('missing step')
212 assert.equal(recorded.outcome, 'insufficient_samples')
213})
Diagnostics4 git calls · 35 ms
CallStatusSizeTime035 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2002.5 KiB · hit29 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=29
git.response.bytes=2592
git.response.cache=HIT
git.response.duration_ms=29
git.fetch.wants=51537cd9bd01ac35ce8f2bbccb338a4438e19eb7
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2592
git.pkt.payload.max=2571
git.sideband.band1.bytes=2570
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e32889cf9c4628
Request
9510bf3e…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format