main
1// A ramp probe is a server response, a client-side loss, or a shed.
2// Loss is not a server error. The measured abort is 1/790 on run 2026-11-01T17-06-08-780Z
3// (web 789ba93, ply 1743461). Run 2026-11-01T19-40-35-485Z (web 22bfb79) stalled the
4// scheduler: 10/128 never answered and both versions returned together at ~69830ms.
5import { none, some, type Option } from '../../src/option.ts'
6import { assertNever } from '../../src/never.ts'
8/** Why an issued probe never produced a status or a ray. `timed_out` is the policy deadline, not a transport error. */
9export type ProbeLossReason = 'aborted' | 'network' | 'timed_out'
11export type ProbeOutcome =
12 | {
13 readonly kind: 'responded'
14 readonly id: string
15 readonly status: number
16 readonly ray: Option<string>
17 readonly latencyMs: number
18 /** The `x-ply-version` header. A lost probe has no version, so this is only on a response. */
19 readonly version: Option<string>
20 }
21 | {
22 readonly kind: 'lost'
23 readonly id: string
24 readonly reason: ProbeLossReason
25 }
26 | {
27 readonly kind: 'shed'
28 readonly id: string
29 }
30 /**
31 * The client was stalled (client-health.ts) while this probe was in flight, or the schedule recorded a stall in its
32 * window (one marker per stall, id `client-stall ...`). Not a loss and not in the denominator: it says nothing about
33 * the Worker. A window with stalls must still reach probe.minSamples real samples or it is `harness_unhealthy`.
34 */
35 | {
36 readonly kind: 'stalled'
37 readonly id: string
38 }
40/** What `fetch` did, before it is a probe outcome. A thrown attempt has no status and no ray. */
41export type FetchAttempt =
42 | {
43 readonly kind: 'response'
44 readonly status: number
45 readonly ray: Option<string>
46 readonly latencyMs: number
47 readonly version: Option<string>
48 }
49 | { readonly kind: 'thrown'; readonly name: string; readonly message: string }
50 | { readonly kind: 'timeout' }
52export type ProbeLoss = {
53 /** Issued probes: responded plus lost. A shed was never sent and is not in this denominator. */
54 readonly total: number
55 readonly count: number
56 readonly rate: number
57 readonly maxLossRate: number
58 /** `rate > maxLossRate`. The loss-rate gate also requires `probe.minSamples` before it can fail a step. */
59 readonly exceeded: boolean
60 /** Responded probes only. A lost probe is not a server error and is not in this count. */
61 readonly responded: number
62 /** HTTP status >= 500 among responded probes. Lost probes are not included. */
63 readonly serverErrors: number
64 readonly ids: readonly string[]
65 readonly timedOut: number
66 readonly aborted: number
67 readonly network: number
68 readonly shed: number
69 /** Probes in flight across a client stall, plus one marker per stall in the window. Not in `total`. */
70 readonly stalled: number
71}
73export type ProbeWindow =
74 | { readonly kind: 'ok'; readonly loss: ProbeLoss }
75 | { readonly kind: 'harness_fault'; readonly loss: ProbeLoss; readonly reason: string }
76 | { readonly kind: 'insufficient_samples'; readonly loss: ProbeLoss; readonly reason: string }
77 | { readonly kind: 'rollback'; readonly loss: ProbeLoss; readonly reason: string }
78 | { readonly kind: 'retry'; readonly loss: ProbeLoss; readonly reason: string }
79 /** The client stalled and the window, even extended, has fewer than minSamples real samples. Prod is not judged. */
80 | { readonly kind: 'harness_unhealthy'; readonly loss: ProbeLoss; readonly reason: string }
82/** Bounds for one probe window. `minSamples` is the loss-rate denominator; `respondedMin` is `burn.minSamples`. */
83export type ProbeJudge = {
84 readonly maxLossRate: number
85 readonly respondedMin: number
86 readonly minSamples: number
87 readonly stallLatencyMs: number
88 readonly fivexxHard: boolean
89}
91/**
92 * Simplified ADT for probe outcomes at the harness edges.
93 * ok | server_error | timeout | shed | transport_error
94 */
95// (simple probe outcome mapping deleted; the ramp judges directly from ProbeOutcome)
97/** One step recorded in ramp.jsonl, events, and the verdict. */
98export type ProbeLossRecord = {
99 readonly percent: number
100 readonly count: number
101 readonly rate: number
102 readonly total: number
103 readonly responded: number
104 readonly ids: readonly string[]
105}
107export function probeLossRecord(percent: number, loss: ProbeLoss): ProbeLossRecord {
108 return {
109 percent,
110 count: loss.count,
111 rate: loss.rate,
112 total: loss.total,
113 responded: loss.responded,
114 ids: loss.ids,
115 }
116}
118function lossReason(name: string, message: string): ProbeLossReason {
119 if (name === 'TimeoutError') return 'timed_out'
120 if (message.includes('timeout')) return 'timed_out'
121 if (name === 'AbortError' || message.includes('aborted')) return 'aborted'
122 return 'network'
123}
125/**
126 * A thrown fetch has no status and no ray, so it is a loss. A policy timeout is
127 * `timed_out`, distinct from a transport error. Status 0 is the same as no HTTP
128 * response. Any other status is a response, ray or not.
129 */
130export function classifyProbe(id: string, attempt: FetchAttempt): ProbeOutcome {
131 switch (attempt.kind) {
132 case 'timeout':
133 return { kind: 'lost', id, reason: 'timed_out' }
134 case 'thrown':
135 return { kind: 'lost', id, reason: lossReason(attempt.name, attempt.message) }
136 case 'response':
137 if (attempt.status === 0) return { kind: 'lost', id, reason: 'network' }
138 return {
139 kind: 'responded',
140 id,
141 status: attempt.status,
142 ray: attempt.ray,
143 latencyMs: attempt.latencyMs,
144 version: attempt.version,
145 }
146 default:
147 return assertNever(attempt)
148 }
149}
151export function emptyProbeLoss(maxLossRate: number): ProbeLoss {
152 return {
153 total: 0,
154 count: 0,
155 rate: 0,
156 maxLossRate,
157 exceeded: false,
158 responded: 0,
159 serverErrors: 0,
160 ids: [],
161 timedOut: 0,
162 aborted: 0,
163 network: 0,
164 shed: 0,
165 stalled: 0,
166 }
167}
169/** Count, rate, and the ids of probes that never got a response. `exceeded` is `rate > maxLossRate`. */
170export function accountProbeLoss(outcomes: readonly ProbeOutcome[], maxLossRate: number): ProbeLoss {
171 const ids: string[] = []
172 let responded = 0
173 let serverErrors = 0
174 let timedOut = 0
175 let aborted = 0
176 let network = 0
177 let shed = 0
178 let stalled = 0
179 for (const outcome of outcomes) {
180 switch (outcome.kind) {
181 case 'lost':
182 ids.push(outcome.id)
183 switch (outcome.reason) {
184 case 'timed_out':
185 timedOut += 1
186 break
187 case 'aborted':
188 aborted += 1
189 break
190 case 'network':
191 network += 1
192 break
193 default:
194 return assertNever(outcome.reason)
195 }
196 break
197 case 'responded':
198 responded += 1
199 if (outcome.status >= 500) serverErrors += 1
200 break
201 case 'shed':
202 shed += 1
203 break
204 case 'stalled':
205 stalled += 1
206 break
207 default:
208 return assertNever(outcome)
209 }
210 }
211 const total = responded + ids.length
212 const count = ids.length
213 const rate = total === 0 ? 0 : count / total
214 return {
215 total,
216 count,
217 rate,
218 maxLossRate,
219 exceeded: rate > maxLossRate,
220 responded,
221 serverErrors,
222 ids,
223 timedOut,
224 aborted,
225 network,
226 shed,
227 stalled,
228 }
229}
231type VersionPace = {
232 readonly slow: number
233 readonly healthy: number
234}
236/** Versions that answered. A version is slow when any of its responses reached the stall latency. */
237function versionPace(outcomes: readonly ProbeOutcome[], stallLatencyMs: number): VersionPace {
238 const slow = new Set<string>()
239 const seen = new Set<string>()
240 for (const outcome of outcomes) {
241 if (outcome.kind !== 'responded' || outcome.version.tag !== 'some') continue
242 const id = outcome.version.value
243 seen.add(id)
244 if (outcome.latencyMs >= stallLatencyMs) slow.add(id)
245 }
246 return { slow: slow.size, healthy: seen.size - slow.size }
247}
249/**
250 * A lost probe has no version. The window is a harness/network stall when the
251 * responded probes do not split by version: both versions show the stall latency,
252 * or both are healthy while the harness timed probes out or shed them. A one-sided
253 * stall stays on the loss-rate gate.
254 */
255function harnessStall(loss: ProbeLoss, pace: VersionPace, judge: ProbeJudge): boolean {
256 if (loss.serverErrors > 0) return false
257 if (pace.slow > 0 && pace.healthy > 0) return false
258 const harnessSide = loss.timedOut > 0 || loss.shed > 0
259 const short = loss.total < judge.minSamples
260 const sameSlow = pace.slow >= 2 && pace.healthy === 0 && loss.count > 0 && (loss.exceeded || short)
261 const sameTimeout = harnessSide && pace.healthy >= 2 && pace.slow === 0 && (short || (loss.exceeded && loss.aborted === 0 && loss.network === 0))
262 const noVersion = pace.slow === 0 && pace.healthy === 0 && harnessSide && (short || loss.exceeded)
263 return sameSlow || sameTimeout || noVersion
264}
266function percent(value: number): string {
267 return (value * 100).toFixed(2)
268}
270/**
271 * A 5xx on a responded probe rolls back when this window would otherwise be
272 * dismissed as a stall or as too small to gate. Loss above `maxLossRate` is a
273 * harness fault only once `total` reaches `minSamples`. A harness stall is a
274 * retry, not a rollback. Fewer responded probes than `respondedMin` is not a pass.
275 * No probes and no sheds (the dwell sent nothing) is not a loss.
276 */
277export function judgeProbeWindow(outcomes: readonly ProbeOutcome[], judge: ProbeJudge): ProbeWindow {
278 const loss = accountProbeLoss(outcomes, judge.maxLossRate)
279 // A stalled client: the schedule already extended the window to replace the stalled time. Short of minSamples real
280 // samples after that, nothing here is evidence about the Worker; with enough, the stalled probes are simply not counted.
281 if (loss.stalled > 0 && loss.total < judge.minSamples) {
282 // A 5xx the Worker served is product evidence whatever the client did.
283 if (loss.serverErrors > 0 && judge.fivexxHard) return { kind: 'rollback', loss, reason: `${loss.serverErrors} 5xx on responded probes` }
284 return {
285 kind: 'harness_unhealthy',
286 loss,
287 reason: `harness unhealthy: the client stalled (${loss.stalled} stalled probes/markers); ${loss.total} real samples < probe.minSamples ${judge.minSamples} after extension`,
288 }
289 }
290 if (loss.total === 0 && loss.shed === 0) return { kind: 'ok', loss }
291 const pace = versionPace(outcomes, judge.stallLatencyMs)
292 const stall = harnessStall(loss, pace, judge)
293 if (loss.serverErrors > 0 && judge.fivexxHard && (stall || loss.total < judge.minSamples)) {
294 return {
295 kind: 'rollback',
296 loss,
297 reason: `${loss.serverErrors} 5xx on responded probes`,
298 }
299 }
300 if (stall) {
301 const rate = percent(loss.rate)
302 const band = pace.slow >= 2 ? 'both versions stalled together' : pace.healthy >= 2 ? 'both versions healthy' : 'no version on the losses'
303 return {
304 kind: 'retry',
305 loss,
306 reason: `harness/network stall: probe loss ${loss.count}/${loss.total} (${rate}%), timed_out ${loss.timedOut}, shed ${loss.shed}; ${band}; re-run the dwell`,
307 }
308 }
309 if (loss.total >= judge.minSamples && loss.exceeded) {
310 return {
311 kind: 'harness_fault',
312 loss,
313 reason: `probe loss ${loss.count}/${loss.total} (${percent(loss.rate)}%) exceeds probe.maxLossRate ${percent(loss.maxLossRate)}%`,
314 }
315 }
316 if (loss.responded < judge.respondedMin) {
317 return {
318 kind: 'insufficient_samples',
319 loss,
320 reason: `insufficient samples: ${loss.responded} responded probes < ${judge.respondedMin}; ${loss.count} lost`,
321 }
322 }
323 return { kind: 'ok', loss }
324}
326/** Header absence for a responded probe. An empty header is none. */
327export function rayOf(header: string | null): Option<string> {
328 if (header === null || header.length === 0) return none()
329 return some(header)
330}
Diagnostics4 git calls · 28 ms
CallStatusSizeTime028 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2003.5 KiB · hit23 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=23 git.response.bytes=3605 git.response.cache=HIT git.response.duration_ms=23 git.fetch.wants=480ad610b0bf8f0bde12bc60245806364a719b4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3605 git.pkt.payload.max=3584 git.sideband.band1.bytes=3583 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2bc704beecfc3
- Request
- 49ed207c…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format