main
1// ply-web#73: a ramp step's latency figures come from its own probes, not from the live smoke that runs inside the 100%
2// step's dwell. Counts, errors, 5xx and burn still read every invocation. Replayed on the measured window of CD run
3// 2026-11-10T14-56-52-021Z (480225db): the step reported p99 696 ms; its probes alone were 14 ms.
4import assert from 'node:assert/strict'
5import { readFileSync } from 'node:fs'
6import { test } from 'node:test'
7import { none, some } from '../src/option.ts'
8import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
9import type { ProbeOutcome } from '../scripts/cd/probe.ts'
10import { ALL_INVOCATIONS, latencySampleOf, observeAfterIngest, runRamp, type LatencySample, type Observation, type RampIo } from '../scripts/cd/ramp.ts'
11import { scopedStats } from '../scripts/cd/stage-ramp.ts'
12import { percentile, rayKey } from '../scripts/cd/telemetry.ts'
14const VERSION = '480225db-d222-405a-a4ca-382bb7b9f4da'
15type Row = readonly [string, number, number, string]
16type Fixture = { readonly recorded: { readonly nextRequests: number; readonly nextWallP99: number; readonly nextCpuP99: number }; readonly probeRays: readonly string[]; readonly rows: readonly Row[] }
17const FIXTURE = JSON.parse(readFileSync(new URL('./fixtures/ramp-100-2026-11-10.json', import.meta.url), 'utf8')) as Fixture
19/** A Workers Observability invocation event, in the shape `statsFromEvents` reads (the fields the real one carries). */
20function invocation(ray: string, wall: number, cpu: number, opts: { readonly outcome: string; readonly status: number } = { outcome: 'ok', status: 200 }): Record<string, unknown> {
21 return {
22 $metadata: { id: `ev-${ray}`, requestId: `req-${ray}`, rayId: ray, type: 'cf-worker-event', service: 'ply', statusCode: opts.status },
23 $workers: { eventType: 'fetch', outcome: opts.outcome, wallTimeMs: wall, cpuTimeMs: cpu, scriptVersion: { id: VERSION }, event: { response: { status: opts.status } } },
24 }
25}
26const responded = (ray: string): ProbeOutcome => ({ kind: 'responded', id: `p-${ray}`, status: 200, ray: some(`${ray}-IAD`), latencyMs: 20, version: some(VERSION) })
28test('latencySampleOf: the responded probes\' rays; lost, shed and ray-less probes are not in it; none at all is every invocation', () => {
29 const outcomes: ProbeOutcome[] = [
30 responded('a585f4159ae599a2'),
31 { kind: 'responded', id: 'no-ray', status: 200, ray: none(), latencyMs: 20, version: some(VERSION) },
32 { kind: 'lost', id: 'lost', reason: 'timed_out' },
33 { kind: 'shed', id: 'shed' },
34 responded('a585f4159ae599a3'),
35 ]
36 assert.deepEqual(latencySampleOf(outcomes), { kind: 'probes', rays: ['a585f4159ae599a2-IAD', 'a585f4159ae599a3-IAD'] })
37 assert.deepEqual(latencySampleOf([{ kind: 'lost', id: 'x', reason: 'network' }]), ALL_INVOCATIONS)
38 assert.deepEqual(latencySampleOf([]), ALL_INVOCATIONS)
39})
41test('rayKey joins a response header ray with the event ray: colo suffix and case do not matter; a non-ray is itself', () => {
42 assert.equal(rayKey('A585F4159AE599A2-IAD'), 'a585f4159ae599a2')
43 assert.equal(rayKey('a585f4159ae599a2'), 'a585f4159ae599a2')
44 assert.equal(rayKey('not-a-ray'), 'not-a-ray')
45})
47test('scopedStats: latency from the probes only; requests, exceptions, errors and 5xx from every invocation', () => {
48 const probes = ['a000000000000001', 'a000000000000002', 'a000000000000003']
49 const events = [
50 invocation(probes[0], 5, 2), invocation(probes[1], 7, 3), invocation(probes[2].toUpperCase(), 9, 4),
51 invocation('b000000000000001', 6580, 14), // a smoke write waiting on Artifacts
52 invocation('b000000000000002', 1200, 30, { outcome: 'exception', status: 500 }), // a smoke request that broke
53 invocation('c000000000000001', 836, 162), // a visual-smoke page
54 ]
55 const sample: LatencySample = { kind: 'probes', rays: probes.map(ray => `${ray}-EWR`) }
56 const scoped = scopedStats(events, sample)
57 assert.deepEqual([...scoped.stats.wallMs].sort((a, b) => a - b), [5, 7, 9])
58 assert.deepEqual([...scoped.stats.cpuMs].sort((a, b) => a - b), [2, 3, 4])
59 assert.equal(scoped.excluded, 3)
60 assert.equal(scoped.landed, 3, 'the probes that reached the telemetry')
61 const whole = scopedStats(events, ALL_INVOCATIONS)
62 assert.equal(whole.excluded, 0)
63 assert.equal(whole.landed, 6)
64 assert.equal(whole.stats.wallMs.length, 6)
65 // the gates read the same counts either way: a smoke failure is still the new version's failure
66 for (const key of ['requests', 'invocations', 'exceptions', 'errors', 'fivexx', 'errorLogs'] as const) {
67 assert.equal(scoped.stats[key], whole.stats[key], key)
68 }
69 assert.equal(scoped.stats.invocations, 6)
70 assert.equal(scoped.stats.exceptions, 1)
71 // probes that never reached the telemetry leave an empty latency sample, not a borrowed one
72 const lost = scopedStats(events, { kind: 'probes', rays: ['d000000000000001-IAD'] })
73 assert.deepEqual(lost.stats.wallMs, [])
74 assert.equal(lost.stats.invocations, 6)
75})
77test('replay, run 2026-11-10T14-56-52-021Z 100% step: every invocation p99 812 ms; the probes\' p99 14 ms', () => {
78 const events = FIXTURE.rows.map(([ray, wall, cpu]) => invocation(ray, wall, cpu))
79 const all = scopedStats(events, ALL_INVOCATIONS)
80 const probes = scopedStats(events, { kind: 'probes', rays: FIXTURE.probeRays })
81 const report = {
82 rows: FIXTURE.rows.length,
83 probeRays: FIXTURE.probeRays.length,
84 recordedAtRun: FIXTURE.recorded,
85 allWallP99: percentile(all.stats.wallMs, 0.99),
86 allCpuP99: percentile(all.stats.cpuMs, 0.99),
87 probeSamples: probes.stats.wallMs.length,
88 probeWallP99: percentile(probes.stats.wallMs, 0.99),
89 probeCpuP99: percentile(probes.stats.cpuMs, 0.99),
90 excluded: probes.excluded,
91 }
92 console.log(JSON.stringify(report))
93 assert.equal(FIXTURE.rows.length, 2091)
94 assert.equal(FIXTURE.probeRays.length, 2001)
95 // the run's own figure (696 ms, from the 2,024 events ingested when it queried) is the same contamination
96 assert.equal(FIXTURE.recorded.nextWallP99, 696)
97 assert.equal(report.allWallP99, 779)
98 assert.equal(report.probeSamples, 1962)
99 assert.equal(report.probeWallP99, 14)
100 assert.equal(report.probeCpuP99, 7)
101 assert.equal(report.excluded, 129, '74 live-smoke requests and 55 visual-smoke pages')
102 assert.equal(probes.landed, 1962)
103 // the ingest wait's stop rule (98% of the 2,000 responded probes) against the run's own snapshot: 2,024 events met
104 // it on the window's total, though only part of them were probes; counted on the sample it waits for the probes
105 assert.ok(FIXTURE.recorded.nextRequests >= 2000 * 0.98)
106 assert.ok(probes.landed >= 2000 * 0.98)
107 // the excluded ones are the slow ones: every invocation over 100 ms is a smoke or visual request
108 const slowProbe = probes.stats.wallMs.filter(ms => ms > 100)
109 assert.deepEqual(slowProbe, [])
110 // every probe invocation in the window is in the sample (the 39 missing of 2,001 never reached the query)
111 assert.ok(probes.stats.invocations === all.stats.invocations)
112})
114function policy(): RolloutPolicy {
115 return parsePolicy(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')).policy
116}
118/** A healthy window routed exactly at `percent` (2,000 invocations). */
119function sampleObservation(percent: number): Observation {
120 const of = (n: number) => ({ requests: n, invocations: n, exceptions: 0, errors: 0, fivexx: 0, errorLogs: 0, wallMs: Array.from({ length: n }, () => 10), cpuMs: Array.from({ length: n }, () => 3), fetchMs: [], v1: [], internal: [] })
121 const newer = Math.round(2000 * percent / 100)
122 return { kind: 'sample', next: of(newer), previous: of(2000 - newer), aggregate: of(2000), landed: 2000 }
123}
125test('runRamp hands each step\'s observation the rays of that step\'s responded probes, and the line names the sample', async () => {
126 const seen: Array<{ readonly percent: number; readonly sample: LatencySample }> = []
127 let clock = 0
128 let round = 0
129 const io: RampIo = {
130 now: () => clock,
131 sleep: async ms => { clock += ms },
132 dwell: async ms => {
133 clock += ms
134 round += 1
135 const outcomes: ProbeOutcome[] = Array.from({ length: 2000 }, (_, i) => responded(`${round.toString(16).padStart(4, '0')}${i.toString(16).padStart(12, '0')}`))
136 outcomes.push({ kind: 'lost', id: 'lost', reason: 'timed_out' })
137 return { outcomes }
138 },
139 applySplit: async () => null,
140 observe: async (_from, _to, percent, sample) => { seen.push({ percent, sample }); return sampleObservation(percent) },
141 rollback: async () => null,
142 }
143 const ramp = await runRamp(policy(), io)
144 assert.equal(ramp.state.kind, 'complete', JSON.stringify(ramp.state))
145 const percents = policy().steps.map(step => step.percent)
146 assert.deepEqual([...new Set(seen.map(s => s.percent))], percents)
147 for (const s of seen) {
148 assert.equal(s.sample.kind, 'probes')
149 if (s.sample.kind === 'probes') {
150 assert.equal(s.sample.rays.length, 2000, 'the responded probes, not the lost one')
151 const step = percents.indexOf(s.percent) + 1
152 assert.ok(s.sample.rays.every(ray => ray.startsWith(step.toString(16).padStart(4, '0'))), 'this step\'s probes only')
153 }
154 }
155 for (const line of ramp.lines) assert.match(line, /latency_sample=probes/)
156})
158test('runRamp with no dwell (no probes) reads latency from every invocation and says so', async () => {
159 const seen: LatencySample[] = []
160 let clock = 0
161 const io: RampIo = {
162 now: () => clock,
163 sleep: async ms => { clock += ms },
164 applySplit: async () => null,
165 observe: async (_from, _to, percent, sample) => { seen.push(sample); return sampleObservation(percent) },
166 rollback: async () => null,
167 }
168 const ramp = await runRamp(policy(), io)
169 assert.equal(ramp.state.kind, 'complete', JSON.stringify(ramp.state))
170 assert.equal(seen.length, policy().steps.length)
171 assert.ok(seen.every(sample => sample.kind === 'all'))
172 for (const line of ramp.lines) assert.match(line, /latency_sample=all/)
173})
175test('the polled ingest wait counts the sample\'s probes, not the live checks\' requests beside them', async () => {
176 const loss = { total: 2000, count: 0, rate: 0, maxLossRate: 0.01, exceeded: false, responded: 2000, serverErrors: 0, ids: [], timedOut: 0, aborted: 0, network: 0, shed: 0, stalled: 0 }
177 // each poll: 2,100 events in the window (smoke and visual included) but only 1,500 probes have landed, then 1,990
178 let polls = 0
179 let clock = 0
180 const io = {
181 now: () => clock,
182 sleep: async (ms: number) => { clock += ms },
183 observe: async (): Promise<Observation> => {
184 const base = sampleObservation(100)
185 const n = polls < 2 ? 1500 : 1990
186 polls += 1
187 return base.kind === 'sample' ? { ...base, aggregate: { ...base.aggregate, invocations: 2100, requests: 2100 }, landed: n } : base
188 },
189 }
190 const observed = await observeAfterIngest(policy(), io, 0, 0, 100, loss, { kind: 'probes', rays: ['a585f4159ae599a2-IAD'] })
191 assert.equal(polls, 3, 'it kept waiting while 2,100 events held only 1,500 probes')
192 assert.equal(observed.kind === 'sample' ? observed.landed : -1, 1990)
193})
Diagnostics4 git calls · 31 ms
CallStatusSizeTime031 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2003.7 KiB · hit26 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=26 git.response.bytes=3807 git.response.cache=HIT git.response.duration_ms=26 git.fetch.wants=1746cda53a2e1a7006781d53fe90e44fcb4f6300 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3807 git.pkt.payload.max=3786 git.sideband.band1.bytes=3785 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2d664ac5c689d
- Request
- 452a9637…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format