main
ply4 git calls59 ms46 KiB3 cachedprotocol v2
330 lines16 KiBRaw
1import { readFile } from 'node:fs/promises'
2import { join } from 'node:path'
3import { SMOKE_LATENCY, TELEMETRY_CHUNK, TELEMETRY_POLLS_MS, TELEMETRY_WAIT_MS } from './baseline.ts'
4import { pollEditSpans, type SpanCheck } from './span-check.ts'
5import { raysByRequestId } from './csv.ts'
6import { asRecord } from './json.ts'
7import { activeLog } from './log.ts'
8import { queryAll, queryCapturedSliced, queryCounts, telemetryQueryRetry, type QueryRetry } from './o11y.ts'
9import { countsFromRows, pollRate, queryRateRows, sendRateProbes, type ProbeResult, type RateVerdict } from './rate-gate.ts'
10import { readTokenFile } from './token.ts'
11import { errorMessage } from './proc.ts'
12import { fail, pass, type StageResult } from './result.ts'
13import { writeJson, writeText } from './run-dir.ts'
14import {
15 applyGates,
16 judgeSmokeTelemetry,
17 lossLine,
18 telemetryLoss,
19 queryFault,
20 queryTo,
21 serviceFilter,
22 spanTypeFilter,
23 summarizeTelemetry,
24 traceIdOf,
25 telemetryMarkdown,
26 telemetryQuery,
27 versionFilter,
28 type Filter,
29 type SmokeRequest,
30 dedupeRows,
31} from './telemetry.ts'
32import type { CdConfig } from './config.ts'
34export type TelemetryValue = {
35 readonly version: string
36 readonly line: string
37 readonly from: number
38 readonly to: number
41function sleep(ms: number): Promise<void> {
42 return new Promise(resolve => setTimeout(resolve, ms))
45export function readSmokeRequests(text: string): SmokeRequest[] {
46 const parsed = JSON.parse(text) as unknown
47 const rec = asRecord(parsed)
48 const requests = rec?.requests
49 if (!Array.isArray(requests)) return []
50 const out: SmokeRequest[] = []
51 for (const item of requests) {
52 const row = asRecord(item)
53 if (!row || typeof row.id !== 'string') continue
54 const expected = Array.isArray(row.expected) ? row.expected.filter((status): status is number => typeof status === 'number') : []
55 const ray = typeof row.ray === 'string' ? row.ray : typeof row.cfRay === 'string' ? row.cfRay : typeof row.cf_ray === 'string' ? row.cf_ray : null
56 out.push({
57 id: row.id,
58 version: typeof row.version === 'string' ? row.version : null,
59 step: typeof row.step === 'string' ? row.step : '?',
60 status: typeof row.status === 'number' ? row.status : 0,
61 expected,
62 ray: ray === '' ? null : ray,
63 })
64 }
65 return out
68function chunks(ids: readonly string[]): string[][] {
69 const out: string[][] = []
70 for (let i = 0; i < ids.length; i += TELEMETRY_CHUNK) out.push(ids.slice(i, i + TELEMETRY_CHUNK))
71 return out
74function csvSibling(reportPath: string): string {
75 return reportPath.endsWith('.json') ? `${reportPath.slice(0, -'.json'.length)}.csv` : `${reportPath}.csv`
78/** The smoke CSV already records each request's cf-ray. Fill a report row that omitted it. */
79async function withRays(reportPath: string, requests: readonly SmokeRequest[]): Promise<SmokeRequest[]> {
80 let rays = new Map<string, string>()
81 try {
82 rays = new Map(raysByRequestId(await readFile(csvSibling(reportPath), 'utf8')))
83 } catch {
84 return [...requests]
85 }
86 return requests.map(request => {
87 if (request.ray) return request
88 const ray = rays.get(request.id)
89 return ray ? { ...request, ray } : request
90 })
93async function logsFor(
94 config: CdConfig,
95 from: number,
96 to: number,
97 ids: readonly string[],
98 version: string | null,
99 retry: QueryRetry,
100): Promise<unknown[]> {
101 const filters: Filter[] = [serviceFilter(config.worker)]
102 if (version) filters.push(versionFilter(version))
103 const out: unknown[] = []
104 for (const group of chunks(ids)) {
105 out.push(...await queryAll('events', from, to, [
106 ...filters,
107 { key: 'msg', operation: 'eq', type: 'string', value: 'ply.request' },
108 { key: 'id', operation: 'in', type: 'string', value: group.join(',') },
109 ], retry))
110 }
111 return out
114/**
115 * Traces-view rows or span events of the smoke's own traces only. An unscoped query over the window pages through every
116 * span the Worker emitted: under load that is over TELEMETRY_MAX_PAGES (run 2026-11-06T21-37-46-183Z: 50 pages in 80 s,
117 * harness_fault). Span events carry no script version, so the scope is the traceIds on the smoke's own log lines.
118 */
119async function byTrace(config: CdConfig, view: 'events' | 'traces', from: number, to: number, traceIds: readonly string[], retry: QueryRetry): Promise<unknown[]> {
120 const base: Filter[] = view === 'events' ? [serviceFilter(config.worker), spanTypeFilter()] : [serviceFilter(config.worker)]
121 const groups = chunks(traceIds)
122 const pages = await Promise.all(groups.map(group => queryAll(view, from, to, [
123 ...base,
124 { key: '$metadata.traceId', operation: 'in', type: 'string', value: group.join(',') },
125 ], retry)))
126 return pages.flat()
129function traceIdsOf(events: readonly unknown[]): string[] {
130 const ids = new Set<string>()
131 for (const item of events) {
132 const event = asRecord(item)
133 if (!event) continue
134 const id = traceIdOf(event)
135 if (id) ids.add(id)
136 }
137 return [...ids]
140/** The span check over the window, starting from the span events already queried; later polls query again. */
141async function editSpans(config: CdConfig, requests: readonly SmokeRequest[], from: number, first: readonly unknown[], retry: QueryRetry, traceIds: readonly string[]): Promise<SpanCheck> {
142 let page: readonly unknown[] | null = first
143 return pollEditSpans(requests, {
144 query: async () => {
145 if (page !== null) {
146 const ready = page
147 page = null
148 return ready
149 }
150 return byTrace(config, 'events', from, queryTo(Date.now()), traceIds, retry)
151 },
152 sleep,
153 now: () => Date.now(),
154 pollsMs: TELEMETRY_POLLS_MS,
155 waitMs: TELEMETRY_WAIT_MS,
156 })
159/**
160 * The rate limiter's silent-degradation gate (scripts/cd/rate-gate.ts): pinned probes as a load client, then the
161 * version's own rate spans since `from`, polled until enough warm default-mode checks are in.
162 */
163export async function rateGate(config: CdConfig, opts: { readonly version: string; readonly from: number; readonly runDir: string; readonly live: boolean }): Promise<{ verdict: RateVerdict; probes: ProbeResult | null }> {
164 const policy = config.policy.rate
165 let probes: ProbeResult | null = null
166 if (policy.probes > 0) {
167 let loadToken: string
168 try {
169 loadToken = await readTokenFile(config.loadTokenFile, 'PLY_LOAD_TOKEN_FILE')
170 } catch (error) {
171 return { verdict: { kind: 'harness_fault', line: `rate: cannot send probes: ${errorMessage(error)}`, counts: null }, probes: null }
172 }
173 const label = `cd-rate:${opts.runDir.split('/').pop() || 'run'}:${opts.live ? 'live' : 'canary'}`
174 probes = await sendRateProbes({ baseUrl: config.baseUrl, worker: config.worker, version: opts.version, loadToken, label, policy }, {
175 fetch: (url, init) => fetch(url, init),
176 sleep,
177 })
178 }
179 const retry = telemetryQueryRetry(config.policy)
180 const verdict = await pollRate(policy, {
181 query: async () => countsFromRows(await queryRateRows(config.worker, opts.version, (filters, groupBys) => queryCounts(opts.from, queryTo(Date.now()), filters, groupBys, retry))),
182 sleep,
183 now: () => Date.now(),
184 pollsMs: TELEMETRY_POLLS_MS,
185 waitMs: TELEMETRY_WAIT_MS,
186 })
187 const probeLine = probes === null ? '' : ` [probes ${probes.answered}/${probes.sent} answered, ${probes.onVersion} on ${opts.version.slice(0, 8)}, ${JSON.stringify(probes.statuses)}]`
188 return { verdict: { ...verdict, line: `${verdict.line}${probeLine}` }, probes }
191export async function stageTelemetry(
192 config: CdConfig,
193 opts: { readonly version: string; readonly since: number; readonly reportPath: string; readonly runDir: string; readonly live: boolean },
194): Promise<StageResult<TelemetryValue>> {
195 try {
196 const requests = await withRays(opts.reportPath, readSmokeRequests(await readFile(opts.reportPath, 'utf8')))
197 const retry = telemetryQueryRetry(config.policy)
198 const from = opts.since
199 // Runs beside the smoke telemetry: its probes and its own ingest wait overlap the smoke's.
200 const rate = rateGate(config, { version: opts.version, from, runDir: opts.runDir, live: opts.live }).catch((error: unknown): { verdict: RateVerdict; probes: null } => ({
201 verdict: { kind: 'harness_fault', line: `rate: telemetry unavailable: ${errorMessage(error)}`, counts: null },
202 probes: null,
203 }))
204 let events: unknown[] = []
205 const started = Date.now()
206 for (let attempt = 0; ; attempt++) {
207 events = await logsFor(config, from, queryTo(Date.now()), requests.map(request => request.id), opts.version, retry)
208 const seen = new Set(events.map(event => {
209 const source = asRecord(asRecord(event)?.source)
210 return typeof source?.id === 'string' ? source.id : ''
211 }).filter(Boolean))
212 const waiting = Date.now() - started
213 if (requests.every(request => seen.has(request.id)) || waiting >= TELEMETRY_WAIT_MS) break
214 const pause = TELEMETRY_POLLS_MS[attempt] ?? TELEMETRY_POLLS_MS[TELEMETRY_POLLS_MS.length - 1] ?? 15_000
215 await sleep(Math.min(pause, TELEMETRY_WAIT_MS - waiting))
216 }
217 const missing = requests.filter(request => !events.some(event => asRecord(asRecord(event)?.source)?.id === request.id)).map(request => request.id)
218 if (missing.length) events.push(...await logsFor(config, from, queryTo(Date.now()), missing, null, retry))
219 const to = queryTo(Date.now())
220 const scoped = [serviceFilter(config.worker), versionFilter(opts.version)]
221 // Sliced: on the live version this is every invocation in the window (≈370 non-span events/s under load, 2026-11-06).
222 const health = (await queryCapturedSliced('events', from, to, scoped, retry)).events
223 // The traces view is one summary row per trace and has no child spans. Fetch durations come from span events.
224 const traceIds = traceIdsOf(events)
225 const [traces, spans] = await Promise.all([
226 byTrace(config, 'traces', from, to, traceIds, retry),
227 byTrace(config, 'events', from, to, traceIds, retry),
228 ])
229 const combined = dedupeRows([...events, ...health])
230 const summary = summarizeTelemetry(opts.version, requests, combined, traces, spans)
231 const { maxLossRate } = config.policy.telemetry
232 const loss = telemetryLoss(requests, summary, maxLossRate)
233 const judged = judgeSmokeTelemetry({ version: opts.version, requests, summary, ceilings: SMOKE_LATENCY, maxLossRate })
234 const judge = applyGates(judged, config.policy.gates)
235 // #67: the pinned smoke's edit-traced request must show ply's spans in Workers tracing. Pinned only: it gates the 0%
236 // canary, and a tracing regression found after 100% is not a reason to roll back.
237 const spanCheck: SpanCheck | null = opts.live ? null : await editSpans(config, requests, from, spans, retry, traceIds)
238 const rateResult = await rate
239 const rateVerdict = rateResult.verdict
240 const stem = opts.live ? 'live-telemetry' : 'telemetry'
241 const spanLine = spanCheck === null ? '' : `\n\n${spanCheck.line}\n`
242 await writeText(opts.runDir, `${stem}.md`, `${telemetryMarkdown({ version: opts.version, from, to, judge, summary, loss })}${spanLine}\n\n${rateVerdict.line}\n`)
243 const failures = judge.kind === 'worker'
244 ? judge.failures.map(failure => {
245 if (failure.kind === 'unexpected-status') return `unexpected-status ${failure.id} ${failure.status}`
246 if (failure.kind === 'latency') return `latency ${failure.metric} ${failure.p} ${failure.value} > ${failure.ceiling}`
247 return `${failure.kind}: ${failure.detail}`
248 })
249 : judge.kind === 'harness_fault' ? [judge.reason] : []
250 // The span check decides only when everything else passed: a worker fault or a harness fault above is reported first.
251 const spanVerdict = judge.kind === 'pass' && spanCheck !== null && spanCheck.kind !== 'pass' ? spanCheck : null
252 if (spanVerdict) failures.push(spanVerdict.line)
253 // The rate gate decides after the smoke and span checks. A rate worker verdict is a silent degradation: canary stops,
254 // live rolls back. A rate harness fault (too few checks ingested, probes unsendable) is unevaluable, not a pass.
255 const rateDecides = judge.kind === 'pass' && spanVerdict === null && rateVerdict.kind !== 'pass' ? rateVerdict : null
256 if (rateDecides) failures.push(rateDecides.line)
257 const kind = spanVerdict ? spanVerdict.kind : rateDecides ? rateDecides.kind : judge.kind
258 const line = `${spanCheck === null ? judge.line : `${judge.line}\n${spanCheck.line}`}\n${rateVerdict.line}`
259 const verdict = {
260 kind,
261 pass: kind === 'pass',
262 version: opts.version,
263 failures,
264 line,
265 loss,
266 spans: spanCheck,
267 rate: { kind: rateVerdict.kind, line: rateVerdict.line, counts: rateVerdict.counts, probes: rateResult.probes },
268 from,
269 to,
270 }
271 await writeJson(opts.runDir, `${stem}.json`, verdict)
272 const log = activeLog()
273 if (log) {
274 await log.observation({
275 measurement: {
276 source: 'telemetry-loss',
277 live: opts.live,
278 count: loss.count,
279 total: loss.total,
280 rate: loss.rate,
281 maxLossRate: loss.maxLossRate,
282 exceeded: loss.exceeded,
283 ids: loss.ids.map(lost => ({ id: lost.id, step: lost.step, log: lost.log, trace: lost.trace })),
284 },
285 query: { logs: telemetryQuery('events', from, to, [serviceFilter(config.worker)], null) },
286 from,
287 to,
288 raw: lossLine(loss),
289 })
290 const ref = await log.observation({
291 measurement: { source: 'smoke-telemetry', pass: judge.kind === 'pass', line: judge.line },
292 query: {
293 traces: telemetryQuery('traces', from, to, [serviceFilter(config.worker)], null),
294 spans: telemetryQuery('events', from, to, [serviceFilter(config.worker), spanTypeFilter()], null),
295 },
296 from,
297 to,
298 raw: JSON.stringify({ line: judge.line, failures }),
299 })
300 await log.decision({
301 inputs: [ref],
302 rules: [{
303 name: judge.kind === 'harness_fault'
304 ? (judge.checks.some(check => check.kind === 'join') ? 'span_join' : 'telemetry_evaluable')
305 : 'smoke_telemetry',
306 value: judge.kind === 'pass' ? 'pass' : judge.kind,
307 }, ...(spanCheck === null ? [] : [{ name: 'ply_spans', value: spanCheck.kind }]), { name: 'rate_degradation', value: rateVerdict.kind }],
308 outcome: kind === 'pass' ? 'advance' : kind === 'harness_fault' ? 'harness_fault' : opts.live ? 'rollback' : 'stop',
309 reason: spanVerdict ? spanVerdict.line : rateDecides ? rateDecides.line : judge.line,
310 })
311 }
312 if (judge.kind === 'harness_fault') return fail('telemetry', judge.reason, false, 'harness_fault')
313 if (judge.kind === 'worker') return fail('telemetry', judge.line)
314 if (spanVerdict && spanVerdict.kind === 'harness_fault') return fail('telemetry', spanVerdict.line, false, 'harness_fault')
315 if (spanVerdict) return fail('telemetry', spanVerdict.line)
316 if (rateDecides && rateDecides.kind === 'harness_fault') return fail('telemetry', rateDecides.line, false, 'harness_fault')
317 if (rateDecides) return fail('telemetry', rateDecides.line)
318 return pass('telemetry', { version: opts.version, line, from, to })
319 } catch (error) {
320 const fault = queryFault(`telemetry unavailable: ${errorMessage(error)}`)
321 await activeLog()?.error(fault.reason, { stage: 'telemetry', fault: fault.kind })
322 const stem = opts.live ? 'live-telemetry' : 'telemetry'
323 await writeJson(opts.runDir, `${stem}.json`, { kind: fault.kind, pass: false, version: opts.version, failures: [fault.reason], line: fault.reason })
324 return fail('telemetry', fault.reason, false, fault.kind)
325 }
328export function reportPath(runDir: string, live: boolean): string {
329 return join(runDir, live ? 'live-smoke.json' : 'smoke.json')
Diagnostics4 git calls · 59 ms
CallStatusSizeTime059 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=4
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=4
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit7 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=7
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2004.9 KiB · hit48 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=48
git.response.bytes=4969
git.response.cache=HIT
git.response.duration_ms=48
git.fetch.wants=97facc846ea1bfb6d5ef9f1203db8ecb02d8d4f3
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4969
git.pkt.payload.max=4948
git.sideband.band1.bytes=4947
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e2fd394a8ccf81
Request
ecb2bff7…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format