main
1// Is a split write being served? After the 0% canary write, pinned requests (Cloudflare-Workers-Version-Overrides) must
2// come back from the canary; after the 100% write, unpinned requests must come back from it. Cloudflare applies an
3// override only on an edge machine that already has the new deployment and otherwise routes by the split, so a check on
4// one keep-alive connection samples one path and can pass while other machines still serve the old version (run
5// 2026-11-10T20-48-18-470Z). Each round here opens PIN_BATCH fresh TCP+TLS connections, and confirmation needs every
6// answer to be the wanted version for PIN_SUSTAIN_MS. Every sample is kept: time since the write, colo, version.
7import { request } from 'node:https'
8import { mapGet } from '../../src/absent.ts'
9import { assertNever } from '../../src/never.ts'
10import { activeLog } from './log.ts'
11import { PIN_BATCH, PIN_INTERVAL_MS, PIN_MAX_MS, PIN_REQUEST_MS, PIN_SUSTAIN_MS } from './baseline.ts'
13export type RouteSample = {
14 /** Milliseconds since the split write (or since the check began when the write time is unknown). */
15 readonly at: number
16 readonly round: number
17 readonly status: number
18 /** `x-ply-version`; null for a lost request or a response without it. */
19 readonly version: string | null
20 /** The colo suffix of cf-ray, or null. */
21 readonly colo: string | null
22}
24/** Why a route check is the Worker's fault. */
25export type WorkerCause =
26 /** The wanted version answered with HTTP 5xx: the product, decided on the first such answer. */
27 | 'served_5xx'
28 /** No answer carried x-ply-version at all. */
29 | 'no_version_header'
31export type RouteVerdict =
32 | { readonly kind: 'confirmed'; readonly samples: readonly RouteSample[]; readonly sustainedFrom: number; readonly at: number }
33 /** Only other versions or lost samples kept the window from sustaining: routing, not the Worker. */
34 | { readonly kind: 'routing'; readonly samples: readonly RouteSample[]; readonly reason: string }
35 /** The wanted version served a 5xx, or nothing carried the version header: the Worker. */
36 | { readonly kind: 'worker'; readonly cause: WorkerCause; readonly samples: readonly RouteSample[]; readonly reason: string }
38/** A verdict that did not confirm. */
39export type RouteFailure = Exclude<RouteVerdict, { readonly kind: 'confirmed' }>
41/** The pipeline fault an unconfirmed verdict carries. Every caller (pin, live, smoke restart) maps it here. */
42export function failureFault(verdict: RouteFailure): 'worker' | 'harness_fault' {
43 switch (verdict.kind) {
44 case 'worker':
45 return 'worker'
46 case 'routing':
47 return 'harness_fault'
48 default:
49 return assertNever(verdict)
50 }
51}
53export type RouteIo = {
54 now(): number
55 sleep(ms: number): Promise<void>
56 /** One GET on a fresh connection. Never throws: a lost request is status 0. */
57 get(url: string, headers: Record<string, string>, timeoutMs: number): Promise<{ readonly status: number; readonly version: string | null; readonly ray: string | null }>
58}
60export type RouteTunables = {
61 readonly batch: number
62 readonly intervalMs: number
63 readonly sustainMs: number
64 readonly maxMs: number
65 readonly requestMs: number
66}
68export const ROUTE_TUNABLES: RouteTunables = {
69 batch: PIN_BATCH,
70 intervalMs: PIN_INTERVAL_MS,
71 sustainMs: PIN_SUSTAIN_MS,
72 maxMs: PIN_MAX_MS,
73 requestMs: PIN_REQUEST_MS,
74}
76/** GET on a new connection (agent: false): no socket is reused, so each sample is its own edge path. */
77export function freshGet(url: string, headers: Record<string, string>, timeoutMs: number): Promise<{ readonly status: number; readonly version: string | null; readonly ray: string | null }> {
78 return new Promise(resolve => {
79 let settled = false
80 const done = (value: { readonly status: number; readonly version: string | null; readonly ray: string | null }): void => {
81 if (settled) return
82 settled = true
83 resolve(value)
84 }
85 const req = request(url, { method: 'GET', headers, agent: false, timeout: timeoutMs }, response => {
86 const version = response.headers['x-ply-version']
87 const ray = response.headers['cf-ray']
88 response.resume()
89 response.on('end', () => {
90 done({
91 status: typeof response.statusCode === 'number' ? response.statusCode : 0,
92 version: typeof version === 'string' && version !== '' ? version : null,
93 ray: typeof ray === 'string' && ray !== '' ? ray : null,
94 })
95 })
96 response.on('error', () => done({ status: 0, version: null, ray: null }))
97 })
98 req.on('timeout', () => {
99 req.destroy(new Error('timeout'))
100 })
101 req.on('error', () => done({ status: 0, version: null, ray: null }))
102 req.end()
103 })
104}
106export const SYSTEM_ROUTE_IO: RouteIo = {
107 now: () => Date.now(),
108 sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),
109 get: freshGet,
110}
112export function coloOf(ray: string | null): string | null {
113 if (ray === null) return null
114 const dash = ray.lastIndexOf('-')
115 return dash < 0 || dash === ray.length - 1 ? null : ray.slice(dash + 1)
116}
118/**
119 * The start of the current all-good run of rounds, or null. A round is good when every sample in it answered 2xx/3xx
120 * from `version`. A lost sample (status 0) is not good: confirmation needs answers, and a stalled client cannot
121 * confirm anything.
122 */
123export function sustainedFrom(samples: readonly RouteSample[], version: string): number | null {
124 const rounds = new Map<number, RouteSample[]>()
125 for (const sample of samples) {
126 const list = mapGet(rounds, sample.round)
127 if (list.tag === 'some') list.value.push(sample)
128 else rounds.set(sample.round, [sample])
129 }
130 let from: number | null = null
131 for (const [, list] of [...rounds.entries()].sort(([a], [b]) => a - b)) {
132 const good = list.length > 0 && list.every(sample => sample.version === version && sample.status >= 200 && sample.status < 400)
133 if (!good) {
134 from = null
135 continue
136 }
137 if (from === null) from = Math.min(...list.map(sample => sample.at))
138 }
139 return from
140}
142/** Offsets (ms since the write) of samples answered by any version other than `version`. */
143export function stragglers(samples: readonly RouteSample[], version: string): readonly RouteSample[] {
144 return samples.filter(sample => sample.version !== null && sample.version !== version)
145}
147export function summarizeRoute(samples: readonly RouteSample[], version: string): string {
148 const late = stragglers(samples, version)
149 const colos = [...new Set(samples.map(sample => sample.colo).filter((colo): colo is string => colo !== null))].sort()
150 const lost = samples.filter(sample => sample.status === 0).length
151 const lastLate = late.length === 0 ? '-' : `${Math.max(...late.map(sample => sample.at))} ms`
152 return `${samples.length} fresh-connection samples, ${late.length} from another version (last at ${lastLate}), ${lost} lost, colos ${colos.join(',') || '-'}`
153}
155/**
156 * Rounds of `batch` fresh-connection GETs every `intervalMs` until every answer has been `version` for `sustainMs`
157 * (measured from the first sample of the first good round to the last sample of the latest), or until `maxMs`.
158 * `since` is the split write's time, so sample offsets are time since the write.
159 */
160export async function confirmRoute(
161 input: { readonly url: string; readonly version: string; readonly headers: Record<string, string>; readonly since: number },
162 io: RouteIo = SYSTEM_ROUTE_IO,
163 tunables: RouteTunables = ROUTE_TUNABLES,
164): Promise<RouteVerdict> {
165 const started = io.now()
166 const samples: RouteSample[] = []
167 let round = 0
168 while (io.now() - started < tunables.maxMs) {
169 round += 1
170 const this1 = round
171 const answers = await Promise.all(Array.from({ length: tunables.batch }, async () => {
172 const answer = await io.get(input.url, input.headers, tunables.requestMs)
173 return { at: io.now() - input.since, round: this1, status: answer.status, version: answer.version, colo: coloOf(answer.ray) }
174 }))
175 samples.push(...answers)
176 // A 5xx from the wanted version is the product (ply-web#146): decided on the first one, never waited out.
177 for (const served of answers) {
178 if (served.version !== input.version || served.status < 500) continue
179 return {
180 kind: 'worker',
181 cause: 'served_5xx',
182 samples,
183 reason: `${input.version} served HTTP ${served.status} at +${served.at} ms (${summarizeRoute(samples, input.version)})`,
184 }
185 }
186 const from = sustainedFrom(samples, input.version)
187 const last = Math.max(...answers.map(sample => sample.at))
188 if (from !== null && last - from >= tunables.sustainMs) return { kind: 'confirmed', samples, sustainedFrom: from, at: last }
189 await io.sleep(tunables.intervalMs)
190 }
191 const answered = samples.filter(sample => sample.status !== 0)
192 const wanted = answered.some(sample => sample.version === input.version)
193 const headerless = answered.length > 0 && answered.every(sample => sample.version === null)
194 if (!wanted && headerless) {
195 return { kind: 'worker', cause: 'no_version_header', samples, reason: `no answer carried x-ply-version in ${tunables.maxMs} ms (${summarizeRoute(samples, input.version)})` }
196 }
197 return { kind: 'routing', samples, reason: `routing: ${input.version} not served on every fresh connection for ${tunables.sustainMs} ms within ${tunables.maxMs} ms (${summarizeRoute(samples, input.version)})` }
198}
200/** Record a confirmation in the run log (every sample, so straggler offsets accumulate run over run) and say what it found. */
201export async function logRoute(
202 input: { readonly purpose: 'pin' | 'live'; readonly version: string; readonly since: number; readonly started: number },
203 verdict: RouteVerdict,
204): Promise<string> {
205 const line = `route-confirm ${input.purpose}: ${verdict.kind} ${input.version.slice(0, 8)}; ${summarizeRoute(verdict.samples, input.version)}${verdict.kind === 'confirmed' ? `; sustained from +${verdict.sustainedFrom} ms to +${verdict.at} ms after the write` : `; ${verdict.reason}`}`
206 const log = activeLog()
207 if (log) {
208 const late = stragglers(verdict.samples, input.version)
209 await log.observation({
210 measurement: {
211 source: 'route-confirm',
212 purpose: input.purpose,
213 kind: verdict.kind,
214 version: input.version,
215 samples: verdict.samples.length,
216 stragglers: late.length,
217 stragglerAtMs: late.map(sample => sample.at),
218 lost: verdict.samples.filter(sample => sample.status === 0).length,
219 colos: [...new Set(verdict.samples.map(sample => sample.colo).filter((colo): colo is string => colo !== null))].sort(),
220 sustainedFromMs: verdict.kind === 'confirmed' ? verdict.sustainedFrom : null,
221 confirmedAtMs: verdict.kind === 'confirmed' ? verdict.at : null,
222 },
223 query: { source: 'fresh-connection GET /', purpose: input.purpose },
224 from: input.started,
225 to: Date.now(),
226 raw: JSON.stringify({ samples: verdict.samples }),
227 })
228 }
229 return line
230}
Diagnostics4 git calls · 595 ms
CallStatusSizeTime0595 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit8 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=8 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2003.6 KiB585 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=585 git.response.bytes=3656 git.response.cache=MISS git.response.duration_ms=585 git.fetch.wants=dcc4a8c93cd2a16b8849aa8291e91665e4375ea3 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3656 git.pkt.payload.max=3635 git.sideband.band1.bytes=3634 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e31ab5cb5f3e6e
- Request
- f37b7b3d…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format