main
1import { createHash } from 'node:crypto'
2import { dig } from '../../src/absent.ts'
3import { assertNever } from '../../src/never.ts'
4import { TELEMETRY_MAX_PAGES, TELEMETRY_PAGE } from './baseline.ts'
5import { runCf } from './cf.ts'
6import { extractJson } from './json.ts'
7import { activeLog } from './log.ts'
8import type { RolloutPolicy } from './policy.ts'
9import { currentLog, recordQueryAttempt, retryQuery, type QueryAttempt, type QueryBound } from './query-retry.ts'
10import { dedupeRows, pageCursor, telemetryQuery, unwrapEvents, type Filter } from './telemetry.ts'
12export type CapturedQuery = {
13 readonly events: unknown[]
14 /**
15 * sha256 of each page's stdout as the run log stored it (its command `stdoutDigest`, blob `blobs/<sha256>.gz`).
16 * An observation cites pages by digest instead of embedding them again.
17 */
18 readonly pages: readonly string[]
19 readonly query: ReturnType<typeof telemetryQuery>
20}
22export type QueryRetry = {
23 readonly bound: QueryBound
24 readonly sleep: (ms: number) => Promise<void>
25 readonly note: (ordinal: number, attempt: QueryAttempt) => Promise<void>
26}
28export function telemetryQueryRetry(policy: RolloutPolicy): QueryRetry {
29 return {
30 bound: { attempts: policy.telemetry.query.attempts, backoffMs: policy.telemetry.query.backoffMs },
31 sleep: ms => new Promise(resolve => {
32 setTimeout(resolve, ms)
33 }),
34 note: (ordinal, attempt) => recordQueryAttempt(currentLog(), ordinal, attempt),
35 }
36}
38async function queryPage(
39 body: object,
40 secrets: readonly string[],
41 retry: QueryRetry,
42): Promise<string> {
43 const result = await retryQuery(retry.bound, {
44 once: async () => {
45 const started = Date.now()
46 const ran = await runCf(['o11y', 'telemetry', 'query', '--body', JSON.stringify(body)], secrets)
47 return { code: ran.code, stdout: ran.stdout, stderr: ran.stderr, latencyMs: Date.now() - started }
48 },
49 sleep: retry.sleep,
50 })
51 let ordinal = 0
52 for (const attempt of result.attempts) {
53 ordinal += 1
54 await retry.note(ordinal, attempt)
55 }
56 switch (result.kind) {
57 case 'ok':
58 return result.stdout
59 case 'lost': {
60 const last = result.attempts[result.attempts.length - 1]
61 const detail = last && last.kind === 'transient' ? last.detail : 'no attempt'
62 throw new Error(`telemetry query failed after ${result.attempts.length} attempts: ${detail}`)
63 }
64 default:
65 return assertNever(result)
66 }
67}
69/** Every page of one observability query, repeats removed, plus each page's capture digest and the first query body. */
70export async function queryCaptured(
71 view: 'events' | 'traces',
72 from: number,
73 to: number,
74 filters: readonly Filter[],
75 retry: QueryRetry,
76 secrets: readonly string[] = [],
77): Promise<CapturedQuery> {
78 const out: unknown[] = []
79 const pages: string[] = []
80 let offset: string | null = null
81 let first: ReturnType<typeof telemetryQuery> | null = null
82 for (let page = 0; page < TELEMETRY_MAX_PAGES; page++) {
83 const body = telemetryQuery(view, from, to, filters, offset)
84 if (!first) first = body
85 const stdout = await queryPage(body, secrets, retry)
86 const log = activeLog()
87 pages.push(log ? log.digest(stdout) : createHash('sha256').update(stdout).digest('hex'))
88 const events = unwrapEvents(extractJson(stdout))
89 out.push(...events)
90 // A short page is the last page. On 2026-11-01 the ramp pages were short of
91 // TELEMETRY_PAGE because the index held that many rows: the series sum equalled
92 // the page, sampleInterval was 1, and statistics.abr_level was 1.
93 if (events.length < TELEMETRY_PAGE) {
94 return { events: dedupeRows(out), pages, query: first }
95 }
96 const cursor = pageCursor(events)
97 if (!cursor) throw new Error('a full telemetry page has no usable $metadata.id cursor')
98 offset = cursor
99 }
100 throw new Error(`more than ${TELEMETRY_MAX_PAGES} pages of telemetry`)
101}
103/** Every page of one observability query. A thrown error means telemetry is unavailable. */
104export async function queryAll(
105 view: 'events' | 'traces',
106 from: number,
107 to: number,
108 filters: readonly Filter[],
109 retry: QueryRetry,
110 secrets: readonly string[] = [],
111): Promise<unknown[]> {
112 return (await queryCaptured(view, from, to, filters, retry, secrets)).events
113}
115/** Width of one slice of a sliced query. 10 s of ply's non-span events under load was 3719 events in 4 pages (2026-11-06). */
116export const SLICE_MS = 10_000
118/** Events that are not child spans: invocations and logs. Spans are ~85% of ply's events under load and gate nothing. */
119export function notSpanFilter(): Filter {
120 return { key: '$metadata.type', operation: 'neq', type: 'string', value: 'span' }
121}
123/**
124 * One query over [from, to], run as parallel `SLICE_MS` windows so a busy window pages in
125 * parallel instead of serially into TELEMETRY_MAX_PAGES. Repeated rows are removed by `rowKey`.
126 */
127export async function queryCapturedSliced(
128 view: 'events' | 'traces',
129 from: number,
130 to: number,
131 filters: readonly Filter[],
132 retry: QueryRetry,
133): Promise<CapturedQuery> {
134 const bounds: [number, number][] = []
135 // The timeframe excludes both endpoints: on 2026-11-08 rows stamped exactly at a slice edge were in neither neighbouring
136 // slice (17:19:44.000Z: 4 rows in [b-1000, b+1], 0 in [b-1000, b] and in [b, b+1000]). End each slice 1 ms past its edge.
137 for (let start = from; start < to; start += SLICE_MS) bounds.push([start, Math.min(to, start + SLICE_MS) + 1])
138 if (bounds.length === 0) bounds.push([from, to])
139 const parts = await Promise.all(bounds.map(([start, end]) => queryCaptured(view, start, end, filters, retry)))
140 const events = dedupeRows(parts.flatMap(part => part.events))
141 const first = parts[0]
142 if (!first) throw new Error('sliced query with no slice')
143 return { events, pages: parts.flatMap(part => part.pages), query: telemetryQuery(view, from, to, filters, null) }
144}
146/** One count per group of a `calculations` aggregate. A group whose key is absent on an event drops that event. */
147export type CountRow = { readonly groups: Readonly<Record<string, string>>; readonly count: number }
149/**
150 * `count` over [from, to], grouped by `groupBys` (string keys). Workers Observability omits an event from a grouped count
151 * when it lacks one of the group keys, so callers group only by keys every matching event carries.
152 */
153export async function queryCounts(from: number, to: number, filters: readonly Filter[], groupBys: readonly string[], retry: QueryRetry): Promise<CountRow[]> {
154 const body = {
155 queryId: 'ply-cd-rate',
156 timeframe: { from, to },
157 view: 'calculations',
158 chartType: 'aggregate',
159 parameters: {
160 calculations: [{ operator: 'count' }],
161 groupBys: groupBys.map(value => ({ type: 'string', value })),
162 limit: 1000,
163 filters: [...filters],
164 },
165 }
166 return countRows(extractJson(await queryPage(body, [], retry)))
167}
169/** The aggregate rows of a calculations response. No calculation block is zero rows, not an error. */
170export function countRows(payload: unknown): CountRow[] {
171 const calcs = dig(payload, ['calculations'])
172 if (calcs.tag === 'none' || !Array.isArray(calcs.value)) throw new Error('calculations response has no calculations list')
173 const found = dig(calcs.value, ['0', 'aggregates'])
174 const aggregates: unknown[] = found.tag === 'some' && Array.isArray(found.value) ? found.value : []
175 const rows: CountRow[] = []
176 for (const item of aggregates) {
177 const groups: Record<string, string> = {}
178 const list = dig(item, ['groups'])
179 if (list.tag === 'some' && Array.isArray(list.value)) {
180 for (const g of list.value as unknown[]) {
181 const key = dig(g, ['key'])
182 const value = dig(g, ['value'])
183 if (key.tag === 'some' && value.tag === 'some') groups[String(key.value)] = String(value.value)
184 }
185 }
186 const count = dig(item, ['count'])
187 rows.push({ groups, count: count.tag === 'some' && typeof count.value === 'number' ? count.value : 0 })
188 }
189 return rows
190}
Diagnostics4 git calls · 228 ms
CallStatusSizeTime0228 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit7 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=7 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2002.9 KiB218 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=218 git.response.bytes=2994 git.response.cache=MISS git.response.duration_ms=218 git.fetch.wants=b14cbd3f3eb4af76d0ac0c8bc42437469904816f git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2994 git.pkt.payload.max=2973 git.sideband.band1.bytes=2972 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e30d99eed98a31
- Request
- 8811acc5…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format