main
ply4 git calls322 ms45 KiB2 cachedprotocol v2
258 lines13 KiBRaw
1// Ramp-only (Ivar, 2026-11-08: "Ramp should be the only option."): 0% canary → pinned smoke → telemetry → the stepped
2// ramp in cd.policy.json → live checks during the 100% step. The direct promote (one write to 100%) is gone; every
3// judgement fix made for it is checked here under the checked-in ramp.
4import assert from 'node:assert/strict'
5import { readFileSync } from 'node:fs'
6import { test } from 'node:test'
7import { none, some } from '../src/option.ts'
8import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
9import { liveVerdict } from '../scripts/cd/pipeline-plan.ts'
10import { planCanary } from '../scripts/cd/deploy-plan.ts'
11import type { ProbeOutcome } from '../scripts/cd/probe.ts'
12import { ALL_INVOCATIONS, INGEST_LANDED, observeAfterIngest, runRamp, type Observation, type RampIo } from '../scripts/cd/ramp.ts'
13import type { VersionStats } from '../scripts/cd/telemetry.ts'
15function checked(): RolloutPolicy {
16 return parsePolicy(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')).policy
19function stats(over: Partial<VersionStats> = {}): VersionStats {
20 const row: VersionStats = {
21 requests: 0,
22 invocations: 0,
23 exceptions: 0,
24 errors: 0,
25 fivexx: 0,
26 errorLogs: 0,
27 wallMs: [],
28 cpuMs: [],
29 fetchMs: [],
30 v1: [],
31 internal: [],
32 ...over,
33 }
34 return { ...row, invocations: 'invocations' in over ? Number(over.invocations) : row.requests }
37function window(next: Partial<VersionStats>, previous: Partial<VersionStats> = {}): Observation {
38 const newer = stats({ wallMs: [10, 10, 10], ...next })
39 const older = stats(previous)
40 return {
41 kind: 'sample',
42 next: newer,
43 previous: older,
44 aggregate: stats({
45 requests: newer.requests + older.requests,
46 invocations: newer.invocations + older.invocations,
47 exceptions: newer.exceptions + older.exceptions,
48 errors: newer.errors + older.errors,
49 fivexx: newer.fivexx + older.fivexx,
50 errorLogs: newer.errorLogs + older.errorLogs,
51 wallMs: [...newer.wallMs, ...older.wallMs],
52 }),
53 landed: newer.invocations + older.invocations,
54 }
57function probes(ok: number, lost = 0): ProbeOutcome[] {
58 const out: ProbeOutcome[] = []
59 for (let i = 0; i < ok; i++) out.push({ kind: 'responded', id: `ok-${i}`, status: 200, ray: none(), latencyMs: 40, version: some('new') })
60 for (let i = 0; i < lost; i++) out.push({ kind: 'lost', id: `lost-${i}`, reason: 'aborted' })
61 return out
64type Trace = { splits: number[]; rolled: number; applied: number[]; observed: number; slept: number[] }
66function io(observe: () => Observation, outcomes: ProbeOutcome[], clock = { t: 0 }): { io: RampIo; trace: Trace } {
67 const trace: Trace = { splits: [], rolled: 0, applied: [], observed: 0, slept: [] }
68 return {
69 trace,
70 io: {
71 now: () => clock.t,
72 sleep: async ms => { trace.slept.push(ms); clock.t += ms },
73 dwell: async ms => { clock.t += ms; return { outcomes } },
74 applySplit: async percent => { trace.splits.push(percent); return null },
75 onApplied: percent => { trace.applied.push(percent) },
76 observe: async () => { trace.observed += 1; return observe() },
77 rollback: async () => { trace.rolled += 1; return null },
78 },
79 }
83function raw(): Record<string, unknown> {
84 return JSON.parse(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')) as Record<string, unknown>
87/** A window whose split matches `percent` of `total`, with optional extras on either side. */
88function at(percent: number, total: number, next: Partial<VersionStats> = {}, previous: Partial<VersionStats> = {}): Observation {
89 const newer = Math.round(total * percent / 100)
90 return window({ requests: newer, ...next }, { requests: total - newer, ...previous })
93test('the checked-in policy ramps: more than one step, the last 100, probe loss deferred to telemetry, ingest polled', () => {
94 const policy = checked()
95 assert.ok(policy.steps.length >= 2)
96 const percents = policy.steps.map(step => step.percent)
97 assert.equal(percents[percents.length - 1], 100)
98 assert.ok(Number(percents[0]) < 100)
99 assert.equal(policy.probe.onHarnessFault, 'telemetry')
100 assert.ok(policy.ingestPollMs > 0)
101 assert.equal('promote' in policy, false)
102})
104test('the policy engine rejects a policy that does not ramp', () => {
105 const single = { ...raw(), steps: [{ percent: 100, dwellMs: 60000 }] }
106 assert.throws(() => parsePolicy(JSON.stringify(single)), /steps: must ramp/)
107 const empty = { ...raw(), steps: [] }
108 assert.throws(() => parsePolicy(JSON.stringify(empty)), /steps: must be a non-empty list/)
109 const missing = raw()
110 delete missing.steps
111 assert.throws(() => parsePolicy(JSON.stringify(missing)), /steps/)
112 const zero = { ...raw(), steps: [{ percent: 0, dwellMs: 1 }, { percent: 100, dwellMs: 1 }] }
113 assert.throws(() => parsePolicy(JSON.stringify(zero)), /steps\[0\]\.percent/)
114 const promote = { ...raw(), promote: { watchMs: 60000, ingestWaitMs: 30000, ingestPollMs: 3000 } }
115 assert.throws(() => parsePolicy(JSON.stringify(promote)), /promote: was removed 2026-11-08/)
116 // Two steps is the least that ramps.
117 const two = { ...raw(), steps: [{ percent: 50, dwellMs: 1 }, { percent: 100, dwellMs: 1 }] }
118 assert.deepEqual(parsePolicy(JSON.stringify(two)).policy.steps.map(step => step.percent), [50, 100])
119})
121test('a clean ramp writes every step in order, calls onApplied per step, and completes', async () => {
122 const policy = checked()
123 let percent = 0
124 const { io: rio, trace } = io(() => at(percent, 2000), probes(2000))
125 const apply = rio.applySplit
126 rio.applySplit = async p => { percent = p; return apply(p) }
127 const result = await runRamp(policy, rio)
128 assert.equal(result.state.kind, 'complete')
129 assert.deepEqual(trace.splits, policy.steps.map(step => step.percent))
130 assert.deepEqual(trace.applied, policy.steps.map(step => step.percent))
131 assert.equal(trace.rolled, 0)
132})
134test('a new-version 5xx rolls back at the first step and at the 100% step (the live watch)', async () => {
135 const policy = checked()
136 for (const bad of [Number(policy.steps.map(step => step.percent)[0]), 100]) {
137 let percent = 0
138 const { io: rio, trace } = io(() => at(percent, 2000, percent === bad ? { fivexx: 3, errors: 3 } : {}), probes(2000))
139 const apply = rio.applySplit
140 rio.applySplit = async p => { percent = p; return apply(p) }
141 const result = await runRamp(policy, rio)
142 assert.equal(result.state.kind, 'rolled-back', `5xx at ${bad}%`)
143 if (result.state.kind === 'rolled-back') assert.equal(result.state.reason, '3 5xx responses on the new version')
144 assert.equal(trace.rolled, 1)
145 assert.equal(trace.splits[trace.splits.length - 1], bad)
146 }
147})
149test('a new-version exception mid-ramp rolls back', async () => {
150 let percent = 0
151 const { io: rio, trace } = io(() => at(percent, 2000, percent === 50 ? { exceptions: 1 } : {}), probes(2000))
152 const apply = rio.applySplit
153 rio.applySplit = async p => { percent = p; return apply(p) }
154 const result = await runRamp(checked(), rio)
155 assert.equal(result.state.kind, 'rolled-back')
156 assert.equal(trace.rolled, 1)
157})
159test('old-version errors do not fail the new release at any step', async () => {
160 // Run 2026-11-06T20-57-44-803Z: "1 5xx response on the old version" rolled back a clean new version.
161 let percent = 0
162 const { io: rio, trace } = io(() => at(percent, 2000, {}, percent < 100 ? { fivexx: 3, exceptions: 1, errorLogs: 2, errors: 3 } : {}), probes(2000))
163 const apply = rio.applySplit
164 rio.applySplit = async p => { percent = p; return apply(p) }
165 const result = await runRamp(checked(), rio)
166 assert.equal(result.state.kind, 'complete')
167 assert.equal(trace.rolled, 0)
168 assert.match(result.lines.join('\n'), /on the old version/)
169})
171test('old-version stragglers at the 100% step are split propagation, informational', async () => {
172 // Runs 2026-11-06T20-01-37-842Z and 20-11-49-685Z: 98.84% and 98.74% at 100% rolled back clean versions.
173 let percent = 0
174 const { io: rio, trace } = io(() => percent === 100 ? window({ requests: 1964 }, { requests: 25, fivexx: 1, errors: 1 }) : at(percent, 2000), probes(2000))
175 const apply = rio.applySplit
176 rio.applySplit = async p => { percent = p; return apply(p) }
177 const result = await runRamp(checked(), rio)
178 assert.equal(result.state.kind, 'complete')
179 assert.equal(trace.rolled, 0)
180 assert.match(result.lines.join('\n'), /split propagation/)
181})
183test('probe-tail loss is the harness under the ramp: recorded, judged on telemetry, not rolled back', async () => {
184 // Runs 2026-11-06T17-32-11-293Z (6/2000) and 18-44-58-475Z (5/2000) rolled back clean versions on this alone.
185 let percent = 0
186 const { io: rio, trace } = io(() => at(percent, 2000), probes(1990, 10))
187 const apply = rio.applySplit
188 rio.applySplit = async p => { percent = p; return apply(p) }
189 const result = await runRamp(checked(), rio)
190 assert.equal(result.state.kind, 'complete')
191 assert.equal(trace.rolled, 0)
192 assert.ok(result.steps.every(step => step.probeLoss.count === 10))
193 // With onHarnessFault rollback (the pre-2026-11-08 file) the same window stopped the ramp.
194 const strict = { ...checked(), probe: { ...checked().probe, onHarnessFault: 'rollback' as const } }
195 const old = io(() => at(10, 2000), probes(1990, 10))
196 assert.equal((await runRamp(strict, old.io)).state.kind, 'stopped')
197})
199test('probe-tail loss beside a new-version 5xx still rolls back', async () => {
200 const { io: rio, trace } = io(() => at(10, 2000, { fivexx: 2, errors: 2 }), probes(1990, 10))
201 const result = await runRamp(checked(), rio)
202 assert.equal(result.state.kind, 'rolled-back')
203 assert.equal(trace.rolled, 1)
204})
206test('a lost telemetry query with no probe sample is a harness fault, distinct from a rollback for cause', async () => {
207 const policy = { ...checked(), probe: { ...checked().probe, rps: 0 } }
208 const trace = { rolled: 0 }
209 const result = await runRamp(policy, {
210 now: () => 0,
211 sleep: async () => {},
212 applySplit: async () => null,
213 observe: async () => ({ kind: 'unavailable', reason: 'telemetry unavailable: timeout' }),
214 rollback: async () => { trace.rolled += 1; return null },
215 })
216 assert.equal(result.state.kind, 'stopped')
217 if (result.state.kind === 'stopped') assert.equal(result.state.fault, 'harness_fault')
218 // Above 0% nothing is left to judge, so traffic goes back to previous, but the verdict stays harness_fault.
219 assert.equal(trace.rolled, 1)
220})
222test('live checks at 100%: a Worker failure (smoke, telemetry, the rate gate, visual) rolls back; a harness fault alone halts', () => {
223 assert.equal(liveVerdict([{ kind: 'pass' }, { kind: 'pass' }, { kind: 'pass' }]), 'done')
224 // stageTelemetry returns a worker fail for a rate-gate degradation (mode, fail_open, dropped flushes) on the live window.
225 assert.equal(liveVerdict([{ kind: 'pass' }, { kind: 'fail', fault: 'worker' }, { kind: 'pass' }]), 'rollback')
226 // Too few rate checks ingested, no Chrome, a lost query: unevaluable, not evidence against the version.
227 assert.equal(liveVerdict([{ kind: 'pass' }, { kind: 'fail', fault: 'harness_fault' }]), 'halt')
228 assert.equal(liveVerdict([{ kind: 'fail', fault: 'harness_fault' }, { kind: 'fail', fault: 'worker' }]), 'rollback')
229})
231test('the polled ingest wait stops as soon as the probes have landed, and gives up at the ceiling', async () => {
232 const policy = checked()
233 const loss = { total: 1000, count: 0, rate: 0, maxLossRate: 0.01, exceeded: false, responded: 1000, serverErrors: 0, ids: [], timedOut: 0, aborted: 0, network: 0, shed: 0, stalled: 0 }
234 let landed = 0
235 const fast = io(() => { landed += 400; return window({ requests: Math.min(landed, 1000) }) }, probes(1000))
236 const observed = await observeAfterIngest(policy, fast.io, 0, 0, 100, loss, ALL_INVOCATIONS)
237 assert.equal(observed.kind, 'sample')
238 if (observed.kind === 'sample') assert.ok(observed.aggregate.invocations >= 1000 * INGEST_LANDED)
239 assert.equal(fast.trace.observed, 3)
240 const slow = io(() => window({ requests: 10 }), probes(1000))
241 await observeAfterIngest(policy, slow.io, 0, 0, 100, loss, ALL_INVOCATIONS)
242 assert.equal(slow.trace.slept.reduce((a, b) => a + b, 0), policy.ingestWaitMs)
243})
245test('deploy-canary refuses a 0% version it did not create unless --replace-canary names it', () => {
246 const live = '11111111-1111-4111-8111-111111111111'
247 const foreign = 'b494b751-0000-4000-8000-000000000000'
248 const next = '22222222-2222-4222-8222-222222222222'
249 const split = [{ versionId: live, percentage: 100 }, { versionId: foreign, percentage: 0 }]
250 const refused = planCanary(split, next)
251 assert.equal(refused.kind, 'conflict')
252 if (refused.kind === 'conflict') assert.match(refused.reason, /--replace-canary b494b751-0000-4000-8000-000000000000/)
253 assert.equal(planCanary(split, next, '33333333-3333-4333-8333-333333333333').kind, 'conflict')
254 assert.deepEqual(planCanary(split, next, foreign), { kind: 'ok', live, dropped: foreign })
255 // Our own version already at 0% (a resumed release) is not a replacement.
256 assert.deepEqual(planCanary([{ versionId: live, percentage: 100 }, { versionId: next, percentage: 0 }], next), { kind: 'ok', live, dropped: null })
257 assert.deepEqual(planCanary([{ versionId: live, percentage: 100 }], next), { kind: 'ok', live, dropped: null })
258})
Diagnostics4 git calls · 322 ms
CallStatusSizeTime0322 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2003.9 KiB317 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=317
git.response.bytes=3964
git.response.cache=MISS
git.response.duration_ms=317
git.fetch.wants=1bb9af5760356dedca2a06ce45b5402e8ac502c1
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3964
git.pkt.payload.max=3943
git.sideband.band1.bytes=3942
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e329043a114628
Request
eaff1b28…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format