main
ply4 git calls203 ms44 KiB2 cachedprotocol v2
239 lines12 KiBRaw
1import assert from 'node:assert/strict'
2import { readFileSync } from 'node:fs'
3import { test } from 'node:test'
4import { FAST_BURN, MIN_SAMPLES } from '../scripts/cd/slo.ts'
5import { applyGates, type SmokeJudge } from '../scripts/cd/telemetry.ts'
6import { DEFAULT_MAX_LOSS_RATE, DEFAULT_POLICY_PATH, loadPolicy, parsePolicy, splitPolicy } from '../scripts/cd/policy.ts'
8function text(): string {
9 return readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')
12function raw(): Record<string, unknown> {
13 return JSON.parse(text()) as Record<string, unknown>
16test('the checked-in policy is the correctness ramp', () => {
17 const loaded = loadPolicy(DEFAULT_POLICY_PATH)
18 assert.equal(loaded.sha256, parsePolicy(text()).sha256)
19 assert.equal(loaded.policy.gates.latency.mode, 'informational')
20 assert.equal(loaded.policy.gates.smoke, 'hard')
21 assert.equal(loaded.policy.gates.fivexx, 'hard')
22 assert.equal(loaded.policy.gates.exceptions, 'hard')
23 assert.equal(loaded.policy.gates.errorLogs, 'hard')
24 assert.equal(loaded.policy.gates.nonOk, 'hard')
25 assert.equal(loaded.policy.gates.burn, 'hard')
26 assert.equal(loaded.policy.ingestWaitMs, 20_000)
27 assert.equal(loaded.policy.warmupMs, 10_000)
28 assert.equal(loaded.policy.probe.rps, 25)
29 assert.deepEqual(loaded.policy.probe.calls.map(call => call.path), [
30 '/',
31 '/v1/discover?remote=https%3A%2F%2F5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net%2Fgit%2Fpublic-v0%2Fply.git',
32 ])
33 assert.deepEqual(loaded.policy.steps.map(step => step.percent), [10, 25, 50, 75, 100])
34 assert.equal(loaded.policy.burn.fastBurn, FAST_BURN)
35 assert.equal(loaded.policy.burn.minSamples, MIN_SAMPLES)
36 assert.equal(loaded.policy.split.z, 3)
37 assert.equal(loaded.policy.split.minNewRequests, MIN_SAMPLES)
38 assert.equal(loaded.policy.split.onMismatch, 'rollback')
39 assert.equal(loaded.policy.split.onInsufficient, 'hold')
40 assert.equal(loaded.policy.probe.maxLossRate, 1 / 790)
41 assert.equal(loaded.policy.probe.onHarnessFault, 'telemetry')
42 assert.equal(loaded.policy.probe.calibration.source, 'run 2026-11-01T17-06-08-780Z')
43 assert.equal(loaded.policy.probe.calibration.date, '2026-11-01')
44 assert.equal(loaded.policy.probe.calibration.window, '100% ramp step')
45 assert.equal(loaded.policy.probe.calibration.n, 790)
46 assert.equal(loaded.policy.probe.calibration.lost, 1)
47 assert.equal(loaded.policy.probe.calibration.web, '789ba93')
48 assert.equal(loaded.policy.probe.calibration.ply, '1743461')
49 assert.equal(loaded.policy.probe.calibration.observedRate, '1/790')
50 assert.equal(loaded.policy.probe.timeoutMs, 2_000)
51 assert.equal(loaded.policy.probe.maxInFlight, 50)
52 assert.equal(loaded.policy.probe.minSamples, 1_000)
53 assert.equal(loaded.policy.probe.stallLatencyMs, 1_000)
54 assert.equal(loaded.policy.probe.windowAttempts, 2)
55 assert.equal(loaded.policy.probe.windowBackoffMs, 1_000)
56 assert.equal(loaded.policy.probe.schedule.source, 'run 2026-11-01T19-40-35-485Z')
57 assert.equal(loaded.policy.probe.schedule.date, '2026-11-01')
58 assert.equal(loaded.policy.probe.schedule.window, '50% ramp step')
59 assert.equal(loaded.policy.probe.schedule.n, 128)
60 assert.equal(loaded.policy.probe.schedule.responded, 118)
61 assert.equal(loaded.policy.probe.schedule.lost, 10)
62 assert.equal(loaded.policy.probe.schedule.healthyN, 1_990)
63 assert.equal(loaded.policy.probe.schedule.web, '22bfb79')
64 assert.equal(loaded.policy.probe.schedule.oldVersion, '4f3d7acf')
65 assert.equal(loaded.policy.probe.schedule.newVersion, '19ee3cab')
66 assert.equal(loaded.policy.probe.schedule.healthyP50Ms, '41-101')
67 assert.equal(loaded.policy.probe.schedule.observedStallMs, 69_830)
68 assert.equal(loaded.policy.probe.schedule.dwellMs, 40_000)
69 assert.equal(loaded.policy.probe.schedule.observedRate, '10/128')
70 assert.match(loaded.policy.probe.schedule.headroom, /PRIOR/)
71 assert.ok(loaded.policy.probe.timeoutMs > 101 * 10)
72 assert.ok(loaded.policy.probe.timeoutMs < loaded.policy.probe.schedule.observedStallMs)
73 assert.ok(loaded.policy.probe.minSamples > loaded.policy.probe.schedule.n)
74 assert.ok(loaded.policy.probe.minSamples < loaded.policy.probe.schedule.healthyN)
75 assert.equal(loaded.policy.cleanup.attempts, 3)
76 assert.equal(loaded.policy.cleanup.backoffMs, 1_000)
77 assert.equal(loaded.policy.cleanup.staleMinutes, 60)
78 assert.equal(loaded.policy.cleanup.calibration.source, 'run 2026-11-01T17-29-43-122Z')
79 assert.equal(loaded.policy.cleanup.calibration.date, '2026-11-01')
80 assert.equal(loaded.policy.cleanup.calibration.window, 'post-ramp scratch repo delete')
81 assert.equal(loaded.policy.cleanup.calibration.n, 1)
82 assert.equal(loaded.policy.cleanup.calibration.latencyMs, 103_000)
83 assert.equal(loaded.policy.telemetry.query.attempts, 3)
84 assert.equal(loaded.policy.telemetry.query.backoffMs, 1_000)
85 assert.equal(loaded.policy.telemetry.query.calibration.source, 'run 2026-11-01T19-17-18-170Z')
86 assert.equal(loaded.policy.telemetry.query.calibration.date, '2026-11-01')
87 assert.equal(loaded.policy.telemetry.query.calibration.window, '100% ramp step')
88 assert.equal(loaded.policy.telemetry.query.calibration.n, 1)
89 assert.equal(loaded.policy.telemetry.query.calibration.web, '07568e0')
90 assert.equal(loaded.policy.telemetry.query.calibration.ply, '1743461')
91 assert.match(loaded.policy.telemetry.query.calibration.observed, /APIError timeout Envelope Error/)
92 assert.match(loaded.policy.telemetry.query.calibration.headroom, /PRIOR/)
93 assert.equal(loaded.policy.cleanup.calibration.web, '9558b10')
94 assert.equal(loaded.policy.cleanup.calibration.namespace, 'ply-loadgen')
95 assert.match(loaded.policy.cleanup.calibration.observed, /404 \[10200\] Repository not found/)
96 assert.match(loaded.policy.cleanup.calibration.headroom, /PRIOR/)
97 assert.deepEqual(loaded.policy.steps.map(step => step.dwellMs), [30_000, 25_000, 25_000, 25_000, 40_000])
98})
100test('a policy that cannot be run is rejected', () => {
101 const decreasing = raw()
102 const steps = decreasing.steps as { percent: number; dwellMs: number }[]
103 steps[1] = { percent: 1, dwellMs: 40_000 }
104 assert.throws(() => parsePolicy(JSON.stringify(decreasing)), /strictly increasing/)
105 const early = raw()
106 ;(early.steps as { percent: number }[]).pop()
107 assert.throws(() => parsePolicy(JSON.stringify(early)), /last step must be 100/)
108 const gate = raw()
109 gate.gates = { ...(gate.gates as Record<string, unknown>), latency: { mode: 'gate' } }
110 assert.throws(() => parsePolicy(JSON.stringify(gate)), /latency=gate needs minSamples, p99Ratio, and effectFloorMs/)
111 const extra = raw()
112 extra.region = 'iad'
113 assert.throws(() => parsePolicy(JSON.stringify(extra)), /unknown field/)
114 assert.throws(() => parsePolicy('{'), /not JSON/)
115 const mismatch = raw()
116 mismatch.split = { ...(mismatch.split as Record<string, unknown>), onMismatch: 'advance' }
117 assert.throws(() => parsePolicy(JSON.stringify(mismatch)), /hold or rollback/)
118 const thin = raw()
119 thin.split = { ...(thin.split as Record<string, unknown>), minNewRequests: 0 }
120 assert.throws(() => parsePolicy(JSON.stringify(thin)), /minNewRequests/)
121 const zee = raw()
122 zee.split = { ...(zee.split as Record<string, unknown>), z: 0 }
123 assert.throws(() => parsePolicy(JSON.stringify(zee)), /split\.z/)
124 const spaced = raw()
125 const spacedCalls = (spaced.probe as { calls: { path: string }[] }).calls
126 const discover = spacedCalls[1]
127 if (!discover) throw new Error('missing discover call')
128 discover.path = '/v1/discover?remote=a b'
129 assert.throws(() => parsePolicy(JSON.stringify(spaced)), /probe\.calls\[1\]\.path/)
130 const bare = raw()
131 const bareCalls = (bare.probe as { calls: { path: string }[] }).calls
132 const root = bareCalls[0]
133 if (!root) throw new Error('missing root call')
134 root.path = 'discover'
135 assert.throws(() => parsePolicy(JSON.stringify(bare)), /must be a path starting with \//)
136})
138test('the checked-in probe can see a correctness regression on the smallest step', () => {
139 const loaded = loadPolicy(DEFAULT_POLICY_PATH).policy
140 const dwellS = loaded.steps[0]?.dwellMs ?? 0
141 const requests = loaded.probe.rps * loaded.probe.calls.length * (dwellS / 1000)
142 const open = loaded.steps.map(step => step.percent).filter(percent => percent < 100)
143 const smallest = Math.min(...open)
144 const expectedNew = requests * (smallest / 100)
145 assert.equal(requests, 1500)
146 assert.equal(smallest, 10)
147 assert.equal(expectedNew, 150)
148 assert.ok(expectedNew >= loaded.split.minNewRequests)
149 const atFloor = 1 - 0.05 ** (1 / loaded.split.minNewRequests)
150 const atExpected = 1 - 0.05 ** (1 / expectedNew)
151 assert.ok(Math.abs(atFloor - 0.0295) < 0.001)
152 assert.ok(Math.abs(atExpected - 0.0198) < 0.001)
153})
155test('every checked-in step reaches its sample floors at the worst measured routed share, and cites its evidence', () => {
156 const loaded = loadPolicy(DEFAULT_POLICY_PATH).policy
157 // Worst new-version share per percent in the ledger (cd.policy.json calibration[steps[*].dwellMs]).
158 const worst: Record<number, number> = { 10: 0.0874, 25: 0.232, 50: 0.4765, 75: 0.734, 100: 0.98 }
159 for (const step of loaded.steps) {
160 const issued = loaded.probe.rps * loaded.probe.calls.length * (step.dwellMs / 1000)
161 assert.ok(issued >= loaded.probe.minSamples, `${step.percent}%: ${issued} issued`)
162 assert.ok(issued >= loaded.burn.minSamples)
163 const share = worst[step.percent]
164 if (share === Number(share)) assert.ok(issued * share >= loaded.split.minNewRequests, `${step.percent}%: ${issued * share} new`)
165 }
166 for (const field of ['steps[*].dwellMs', 'ingestPollMs', 'ingestWaitMs']) {
167 assert.ok(field in loaded.calibration, `calibration for ${field}`)
168 }
169 const bad = { ...raw(), calibration: { 'promote.watchMs': { source: 's', date: 'd', window: 'w', n: 1, observed: 'o', headroom: 'h' } } }
170 assert.throws(() => parsePolicy(JSON.stringify(bad)), /calibration\.promote\.watchMs: names no schedule field/)
171})
173test('latency=gate with thresholds is a policy, and warm-up may be omitted', () => {
174 const body = raw()
175 body.gates = {
176 ...(body.gates as Record<string, unknown>),
177 latency: { mode: 'gate', minSamples: 20, p99Ratio: 2, effectFloorMs: 10 },
178 }
179 delete body.warmupMs
180 const loaded = parsePolicy(JSON.stringify(body))
181 assert.equal(loaded.policy.gates.latency.mode, 'gate')
182 assert.equal(loaded.policy.warmupMs, 0)
183 if (loaded.policy.gates.latency.mode === 'gate') assert.equal(loaded.policy.gates.latency.p99Ratio, 2)
184})
186test('--policy is taken off the command line', () => {
187 assert.deepEqual(splitPolicy(['--policy', 'other.json', 'cd', '--no-bump']), {
188 argv: ['cd', '--no-bump'],
189 policyPath: 'other.json',
190 })
191 assert.deepEqual(splitPolicy(['cd']), { argv: ['cd'], policyPath: null })
192 const missing = splitPolicy(['--policy'])
193 assert.equal('error' in missing, true)
194 const twice = splitPolicy(['--policy', 'a.json', '--policy', 'b.json'])
195 assert.equal('error' in twice, true)
196})
198test('informational latency does not fail smoke telemetry', () => {
199 const verdict = {
200 pass: true,
201 total: 1,
202 matched: 1,
203 allowance: 2,
204 lost: [],
205 unconfirmed: [],
206 otherVersion: [],
207 reasons: [],
208 line: 'telemetry: 1/1 matched',
209 }
210 const judge: SmokeJudge = {
211 kind: 'worker',
212 line: 'telemetry: 1/1 matched; latency',
213 failures: [{ kind: 'latency', metric: 'wall', p: 'p99', value: 30_000, ceiling: 25_000 }],
214 verdict,
215 }
216 const gates = loadPolicy(DEFAULT_POLICY_PATH).policy.gates
217 const applied = applyGates(judge, gates)
218 assert.equal(applied.kind, 'pass')
219 if (applied.kind === 'pass') assert.match(applied.line, /informational:/)
220 const gated = applyGates(judge, {
221 ...gates,
222 latency: { mode: 'gate', minSamples: 20, p99Ratio: 2, effectFloorMs: 10 },
223 })
224 assert.equal(gated.kind, 'worker')
225})
227test('telemetry.maxLossRate is validated and defaults when omitted', () => {
228 const base = JSON.parse(text()) as Record<string, unknown>
229 assert.equal(parsePolicy(text()).policy.telemetry.maxLossRate, DEFAULT_MAX_LOSS_RATE)
230 const { telemetry: _omitted, ...without } = base
231 assert.equal(parsePolicy(JSON.stringify(without)).policy.telemetry.maxLossRate, DEFAULT_MAX_LOSS_RATE)
232 const tight = 0.1
233 const query = (base.telemetry as { query: unknown }).query
234 assert.equal(parsePolicy(JSON.stringify({ ...base, telemetry: { maxLossRate: tight, query } })).policy.telemetry.maxLossRate, tight)
235 for (const bad of [0, -0.1, 1.5, '0.5', null]) {
236 assert.throws(() => parsePolicy(JSON.stringify({ ...base, telemetry: { maxLossRate: bad, query } })), /telemetry.maxLossRate: must be a number in \(0, 1\]/)
237 }
238 assert.throws(() => parsePolicy(JSON.stringify({ ...base, telemetry: { maxLoss: 1 } })), /telemetry.maxLoss: unknown field/)
239})
Diagnostics4 git calls · 203 ms
CallStatusSizeTime0203 ms
discover200152 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit13 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=13
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=13
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2003.4 KiB188 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=188
git.response.bytes=3462
git.response.cache=MISS
git.response.duration_ms=188
git.fetch.wants=5e5ca6087ccf83981461adfa55b294e4e444792c
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3462
git.pkt.payload.max=3441
git.sideband.band1.bytes=3440
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e2fd974c59cf81
Request
8a4eb695…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format