main
ply4 git calls31 ms44 KiB3 cachedprotocol v2
193 lines11 KiBRaw
1// ply-web#73: a ramp step's latency figures come from its own probes, not from the live smoke that runs inside the 100%
2// step's dwell. Counts, errors, 5xx and burn still read every invocation. Replayed on the measured window of CD run
3// 2026-11-10T14-56-52-021Z (480225db): the step reported p99 696 ms; its probes alone were 14 ms.
4import assert from 'node:assert/strict'
5import { readFileSync } from 'node:fs'
6import { test } from 'node:test'
7import { none, some } from '../src/option.ts'
8import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
9import type { ProbeOutcome } from '../scripts/cd/probe.ts'
10import { ALL_INVOCATIONS, latencySampleOf, observeAfterIngest, runRamp, type LatencySample, type Observation, type RampIo } from '../scripts/cd/ramp.ts'
11import { scopedStats } from '../scripts/cd/stage-ramp.ts'
12import { percentile, rayKey } from '../scripts/cd/telemetry.ts'
14const VERSION = '480225db-d222-405a-a4ca-382bb7b9f4da'
15type Row = readonly [string, number, number, string]
16type Fixture = { readonly recorded: { readonly nextRequests: number; readonly nextWallP99: number; readonly nextCpuP99: number }; readonly probeRays: readonly string[]; readonly rows: readonly Row[] }
17const FIXTURE = JSON.parse(readFileSync(new URL('./fixtures/ramp-100-2026-11-10.json', import.meta.url), 'utf8')) as Fixture
19/** A Workers Observability invocation event, in the shape `statsFromEvents` reads (the fields the real one carries). */
20function invocation(ray: string, wall: number, cpu: number, opts: { readonly outcome: string; readonly status: number } = { outcome: 'ok', status: 200 }): Record<string, unknown> {
21 return {
22 $metadata: { id: `ev-${ray}`, requestId: `req-${ray}`, rayId: ray, type: 'cf-worker-event', service: 'ply', statusCode: opts.status },
23 $workers: { eventType: 'fetch', outcome: opts.outcome, wallTimeMs: wall, cpuTimeMs: cpu, scriptVersion: { id: VERSION }, event: { response: { status: opts.status } } },
24 }
26const responded = (ray: string): ProbeOutcome => ({ kind: 'responded', id: `p-${ray}`, status: 200, ray: some(`${ray}-IAD`), latencyMs: 20, version: some(VERSION) })
28test('latencySampleOf: the responded probes\' rays; lost, shed and ray-less probes are not in it; none at all is every invocation', () => {
29 const outcomes: ProbeOutcome[] = [
30 responded('a585f4159ae599a2'),
31 { kind: 'responded', id: 'no-ray', status: 200, ray: none(), latencyMs: 20, version: some(VERSION) },
32 { kind: 'lost', id: 'lost', reason: 'timed_out' },
33 { kind: 'shed', id: 'shed' },
34 responded('a585f4159ae599a3'),
35 ]
36 assert.deepEqual(latencySampleOf(outcomes), { kind: 'probes', rays: ['a585f4159ae599a2-IAD', 'a585f4159ae599a3-IAD'] })
37 assert.deepEqual(latencySampleOf([{ kind: 'lost', id: 'x', reason: 'network' }]), ALL_INVOCATIONS)
38 assert.deepEqual(latencySampleOf([]), ALL_INVOCATIONS)
39})
41test('rayKey joins a response header ray with the event ray: colo suffix and case do not matter; a non-ray is itself', () => {
42 assert.equal(rayKey('A585F4159AE599A2-IAD'), 'a585f4159ae599a2')
43 assert.equal(rayKey('a585f4159ae599a2'), 'a585f4159ae599a2')
44 assert.equal(rayKey('not-a-ray'), 'not-a-ray')
45})
47test('scopedStats: latency from the probes only; requests, exceptions, errors and 5xx from every invocation', () => {
48 const probes = ['a000000000000001', 'a000000000000002', 'a000000000000003']
49 const events = [
50 invocation(probes[0], 5, 2), invocation(probes[1], 7, 3), invocation(probes[2].toUpperCase(), 9, 4),
51 invocation('b000000000000001', 6580, 14), // a smoke write waiting on Artifacts
52 invocation('b000000000000002', 1200, 30, { outcome: 'exception', status: 500 }), // a smoke request that broke
53 invocation('c000000000000001', 836, 162), // a visual-smoke page
54 ]
55 const sample: LatencySample = { kind: 'probes', rays: probes.map(ray => `${ray}-EWR`) }
56 const scoped = scopedStats(events, sample)
57 assert.deepEqual([...scoped.stats.wallMs].sort((a, b) => a - b), [5, 7, 9])
58 assert.deepEqual([...scoped.stats.cpuMs].sort((a, b) => a - b), [2, 3, 4])
59 assert.equal(scoped.excluded, 3)
60 assert.equal(scoped.landed, 3, 'the probes that reached the telemetry')
61 const whole = scopedStats(events, ALL_INVOCATIONS)
62 assert.equal(whole.excluded, 0)
63 assert.equal(whole.landed, 6)
64 assert.equal(whole.stats.wallMs.length, 6)
65 // the gates read the same counts either way: a smoke failure is still the new version's failure
66 for (const key of ['requests', 'invocations', 'exceptions', 'errors', 'fivexx', 'errorLogs'] as const) {
67 assert.equal(scoped.stats[key], whole.stats[key], key)
68 }
69 assert.equal(scoped.stats.invocations, 6)
70 assert.equal(scoped.stats.exceptions, 1)
71 // probes that never reached the telemetry leave an empty latency sample, not a borrowed one
72 const lost = scopedStats(events, { kind: 'probes', rays: ['d000000000000001-IAD'] })
73 assert.deepEqual(lost.stats.wallMs, [])
74 assert.equal(lost.stats.invocations, 6)
75})
77test('replay, run 2026-11-10T14-56-52-021Z 100% step: every invocation p99 812 ms; the probes\' p99 14 ms', () => {
78 const events = FIXTURE.rows.map(([ray, wall, cpu]) => invocation(ray, wall, cpu))
79 const all = scopedStats(events, ALL_INVOCATIONS)
80 const probes = scopedStats(events, { kind: 'probes', rays: FIXTURE.probeRays })
81 const report = {
82 rows: FIXTURE.rows.length,
83 probeRays: FIXTURE.probeRays.length,
84 recordedAtRun: FIXTURE.recorded,
85 allWallP99: percentile(all.stats.wallMs, 0.99),
86 allCpuP99: percentile(all.stats.cpuMs, 0.99),
87 probeSamples: probes.stats.wallMs.length,
88 probeWallP99: percentile(probes.stats.wallMs, 0.99),
89 probeCpuP99: percentile(probes.stats.cpuMs, 0.99),
90 excluded: probes.excluded,
91 }
92 console.log(JSON.stringify(report))
93 assert.equal(FIXTURE.rows.length, 2091)
94 assert.equal(FIXTURE.probeRays.length, 2001)
95 // the run's own figure (696 ms, from the 2,024 events ingested when it queried) is the same contamination
96 assert.equal(FIXTURE.recorded.nextWallP99, 696)
97 assert.equal(report.allWallP99, 779)
98 assert.equal(report.probeSamples, 1962)
99 assert.equal(report.probeWallP99, 14)
100 assert.equal(report.probeCpuP99, 7)
101 assert.equal(report.excluded, 129, '74 live-smoke requests and 55 visual-smoke pages')
102 assert.equal(probes.landed, 1962)
103 // the ingest wait's stop rule (98% of the 2,000 responded probes) against the run's own snapshot: 2,024 events met
104 // it on the window's total, though only part of them were probes; counted on the sample it waits for the probes
105 assert.ok(FIXTURE.recorded.nextRequests >= 2000 * 0.98)
106 assert.ok(probes.landed >= 2000 * 0.98)
107 // the excluded ones are the slow ones: every invocation over 100 ms is a smoke or visual request
108 const slowProbe = probes.stats.wallMs.filter(ms => ms > 100)
109 assert.deepEqual(slowProbe, [])
110 // every probe invocation in the window is in the sample (the 39 missing of 2,001 never reached the query)
111 assert.ok(probes.stats.invocations === all.stats.invocations)
112})
114function policy(): RolloutPolicy {
115 return parsePolicy(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')).policy
118/** A healthy window routed exactly at `percent` (2,000 invocations). */
119function sampleObservation(percent: number): Observation {
120 const of = (n: number) => ({ requests: n, invocations: n, exceptions: 0, errors: 0, fivexx: 0, errorLogs: 0, wallMs: Array.from({ length: n }, () => 10), cpuMs: Array.from({ length: n }, () => 3), fetchMs: [], v1: [], internal: [] })
121 const newer = Math.round(2000 * percent / 100)
122 return { kind: 'sample', next: of(newer), previous: of(2000 - newer), aggregate: of(2000), landed: 2000 }
125test('runRamp hands each step\'s observation the rays of that step\'s responded probes, and the line names the sample', async () => {
126 const seen: Array<{ readonly percent: number; readonly sample: LatencySample }> = []
127 let clock = 0
128 let round = 0
129 const io: RampIo = {
130 now: () => clock,
131 sleep: async ms => { clock += ms },
132 dwell: async ms => {
133 clock += ms
134 round += 1
135 const outcomes: ProbeOutcome[] = Array.from({ length: 2000 }, (_, i) => responded(`${round.toString(16).padStart(4, '0')}${i.toString(16).padStart(12, '0')}`))
136 outcomes.push({ kind: 'lost', id: 'lost', reason: 'timed_out' })
137 return { outcomes }
138 },
139 applySplit: async () => null,
140 observe: async (_from, _to, percent, sample) => { seen.push({ percent, sample }); return sampleObservation(percent) },
141 rollback: async () => null,
142 }
143 const ramp = await runRamp(policy(), io)
144 assert.equal(ramp.state.kind, 'complete', JSON.stringify(ramp.state))
145 const percents = policy().steps.map(step => step.percent)
146 assert.deepEqual([...new Set(seen.map(s => s.percent))], percents)
147 for (const s of seen) {
148 assert.equal(s.sample.kind, 'probes')
149 if (s.sample.kind === 'probes') {
150 assert.equal(s.sample.rays.length, 2000, 'the responded probes, not the lost one')
151 const step = percents.indexOf(s.percent) + 1
152 assert.ok(s.sample.rays.every(ray => ray.startsWith(step.toString(16).padStart(4, '0'))), 'this step\'s probes only')
153 }
154 }
155 for (const line of ramp.lines) assert.match(line, /latency_sample=probes/)
156})
158test('runRamp with no dwell (no probes) reads latency from every invocation and says so', async () => {
159 const seen: LatencySample[] = []
160 let clock = 0
161 const io: RampIo = {
162 now: () => clock,
163 sleep: async ms => { clock += ms },
164 applySplit: async () => null,
165 observe: async (_from, _to, percent, sample) => { seen.push(sample); return sampleObservation(percent) },
166 rollback: async () => null,
167 }
168 const ramp = await runRamp(policy(), io)
169 assert.equal(ramp.state.kind, 'complete', JSON.stringify(ramp.state))
170 assert.equal(seen.length, policy().steps.length)
171 assert.ok(seen.every(sample => sample.kind === 'all'))
172 for (const line of ramp.lines) assert.match(line, /latency_sample=all/)
173})
175test('the polled ingest wait counts the sample\'s probes, not the live checks\' requests beside them', async () => {
176 const loss = { total: 2000, count: 0, rate: 0, maxLossRate: 0.01, exceeded: false, responded: 2000, serverErrors: 0, ids: [], timedOut: 0, aborted: 0, network: 0, shed: 0, stalled: 0 }
177 // each poll: 2,100 events in the window (smoke and visual included) but only 1,500 probes have landed, then 1,990
178 let polls = 0
179 let clock = 0
180 const io = {
181 now: () => clock,
182 sleep: async (ms: number) => { clock += ms },
183 observe: async (): Promise<Observation> => {
184 const base = sampleObservation(100)
185 const n = polls < 2 ? 1500 : 1990
186 polls += 1
187 return base.kind === 'sample' ? { ...base, aggregate: { ...base.aggregate, invocations: 2100, requests: 2100 }, landed: n } : base
188 },
189 }
190 const observed = await observeAfterIngest(policy(), io, 0, 0, 100, loss, { kind: 'probes', rays: ['a585f4159ae599a2-IAD'] })
191 assert.equal(polls, 3, 'it kept waiting while 2,100 events held only 1,500 probes')
192 assert.equal(observed.kind === 'sample' ? observed.landed : -1, 1990)
193})
Diagnostics4 git calls · 31 ms
CallStatusSizeTime031 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2003.7 KiB · hit26 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=26
git.response.bytes=3807
git.response.cache=HIT
git.response.duration_ms=26
git.fetch.wants=1746cda53a2e1a7006781d53fe90e44fcb4f6300
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3807
git.pkt.payload.max=3786
git.sideband.band1.bytes=3785
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e2d664ac5c689d
Request
452a9637…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format