main
ply4 git calls25 ms45 KiB3 cachedprotocol v2
234 lines12 KiBRaw
1// scripts/calibrate: input distributions from Workers Observability, per route (issue #73).
2//
3// node --experimental-strip-types scripts/calibrate/main.ts --since 7d
4// node --experimental-strip-types scripts/calibrate/main.ts --from 2026-11-01T00:00:00Z --to 2026-11-02T00:00:00Z --route 'GET /v1/tree'
5// node --experimental-strip-types scripts/calibrate/main.ts --since 7d --check --report drift.md
6// node --experimental-strip-types scripts/calibrate/main.ts --fixture test/fixtures/calibrate/spans.json --check
7// node --experimental-strip-types scripts/calibrate/main.ts --since 72h --include-noisy # unfiltered; pair with a smaller --slice
8//
9// Prints, per route (and `*` for all routes), every numeric input attribute's n, p50/p90/p99/p99.9, max and log2
10// histogram, and categorical counts (error kind, cache status, outcome). --check judges every threshold in
11// scripts/calibrate/thresholds.json that names a metric against this window and fails (exit 1) when one has left its
12// band, or when a threshold lacks provenance. --cite KEY prints a citation block for that threshold from this window.
13// --fixture reads recorded query responses instead of querying (no credentials, no network).
14//
15// Exit: 0 in band, 1 out of band or a provenance problem, 2 a usage or query error.
16import { readFileSync, writeFileSync } from 'node:fs'
17import { atIndex, captureGroup, mapGet, recordGet } from '../../src/absent.ts'
18import { orNull, unwrapOr } from '../../src/option.ts'
19import { WORKER_DEFAULT } from '../cd/baseline.ts'
20import { extractJson } from '../cd/json.ts'
21import { telemetryQueryRetry } from '../cd/o11y.ts'
22import { DEFAULT_POLICY_PATH, loadPolicy } from '../cd/policy.ts'
23import { unwrapEvents } from '../cd/telemetry.ts'
24import { currentThresholds, loadRegistry, REGISTRY_PATH } from './current.ts'
25import { checkBands, citationFor, isOutOfBand, provenanceProblems, type BandResult, type Problem } from './provenance.ts'
26import { citedQuery, querySpanEvents, type SpanQuery } from './query.ts'
27import { calibrationStats, statsText, type CalibrationStats } from './stats.ts'
29const UNIT_MS: Readonly<Record<string, number>> = { m: 60_000, h: 3_600_000, d: 86_400_000 }
31/** `30m`, `12h`, `7d` in milliseconds. */
32export function parseDuration(text: string): number {
33 const match = /^(\d+)([mhd])$/.exec(text)
34 const unit = recordGet(UNIT_MS, unwrapOr(captureGroup(match, 2), ''))
35 if (unit.tag === 'none') throw new Error(`duration ${JSON.stringify(text)} is not <n>m, <n>h or <n>d`)
36 const n = Number(unwrapOr(captureGroup(match, 1), ''))
37 if (!(n > 0)) throw new Error(`duration ${text} must be positive`)
38 return n * unit.value
41export type Args = {
42 readonly from: number
43 readonly to: number
44 readonly worker: string
45 readonly sliceMs: number
46 readonly routes: readonly string[] | null
47 readonly attributes: readonly string[] | null
48 readonly versions: readonly string[] | null
49 readonly fixtures: readonly string[]
50 readonly histograms: boolean
51 readonly check: boolean
52 readonly report: string | null
53 readonly json: string | null
54 readonly cite: readonly string[]
55 readonly registry: string
56 readonly policy: string
57 /** When true, the observability query does not allowlist routes (default is /v1 + GET /). */
58 readonly includeNoisy: boolean
61/**
62 * Ten minutes per slice by default (loop 2026-11-08): with DEFAULT_CALIBRATE_ROUTES, a peak hour still
63 * had ~61k allowlisted spans (discover 26k + GET / 17k + read 9k + tree 6k during a load burst), which
64 * overflows TELEMETRY_MAX_PAGES (50×1000). Ten-minute slices of that hour stayed ≤16,885. Pass
65 * `--include-noisy` only with an even smaller `--slice` when you need viewer/rate.* distributions.
66 */
67const DEFAULT_SLICE = '10m'
69/** `first` when given, else `fallback`. */
70function or(first: string | null, fallback: string): string {
71 return first === null ? fallback : first
74export function parseArgs(argv: readonly string[], now: number, env: Readonly<Record<string, unknown>>): Args {
75 const many = new Map<string, string[]>()
76 const flags = new Set<string>()
77 const BOOL = new Set(['--check', '--no-histogram', '--include-noisy'])
78 for (let i = 0; i < argv.length; i++) {
79 const arg = unwrapOr(atIndex(argv, i), '')
80 if (BOOL.has(arg)) {
81 flags.add(arg)
82 continue
83 }
84 if (!arg.startsWith('--')) throw new Error(`unexpected argument ${arg}`)
85 const value = atIndex(argv, i + 1)
86 if (value.tag === 'none') throw new Error(`${arg} needs a value`)
87 many.set(arg, [...unwrapOr(mapGet(many, arg), []), value.value])
88 i += 1
89 }
90 const known = new Set(['--since', '--from', '--to', '--worker', '--slice', '--route', '--attr', '--version', '--fixture', '--report', '--json', '--cite', '--registry', '--policy'])
91 for (const key of many.keys()) if (!known.has(key)) throw new Error(`unknown option ${key}`)
92 const one = (key: string): string | null => {
93 const values = unwrapOr(mapGet(many, key), [])
94 if (values.length > 1) throw new Error(`${key} given more than once`)
95 return orNull(atIndex(values, 0))
96 }
97 const list = (key: string): readonly string[] | null => orNull(mapGet(many, key))
98 const fromText = one('--from')
99 const toText = one('--to')
100 const since = one('--since')
101 const to = toText === null ? now : Date.parse(toText)
102 if (!Number.isFinite(to)) throw new Error(`--to ${toText} is not a date`)
103 let from: number
104 if (fromText !== null) {
105 if (since !== null) throw new Error('--from and --since are exclusive')
106 from = Date.parse(fromText)
107 if (!Number.isFinite(from)) throw new Error(`--from ${fromText} is not a date`)
108 } else {
109 from = to - parseDuration(or(since, '1d'))
110 }
111 if (!(to > from)) throw new Error('the window must end after it starts')
112 return {
113 from,
114 to,
115 worker: or(one('--worker'), envText(env, 'PLY_WORKER', WORKER_DEFAULT)),
116 sliceMs: parseDuration(or(one('--slice'), DEFAULT_SLICE)),
117 routes: list('--route'),
118 attributes: list('--attr'),
119 versions: list('--version'),
120 fixtures: unwrapOr(mapGet(many, '--fixture'), []),
121 histograms: !flags.has('--no-histogram'),
122 check: flags.has('--check'),
123 report: one('--report'),
124 json: one('--json'),
125 cite: unwrapOr(mapGet(many, '--cite'), []),
126 registry: or(one('--registry'), REGISTRY_PATH),
127 policy: or(one('--policy'), DEFAULT_POLICY_PATH),
128 includeNoisy: flags.has('--include-noisy'),
129 }
132function envText(env: Readonly<Record<string, unknown>>, key: string, fallback: string): string {
133 const found = recordGet(env, key)
134 return found.tag === 'some' && typeof found.value === 'string' && found.value !== '' ? found.value : fallback
137/** Recorded query responses (cf's stdout, banner and all), concatenated. */
138export function fixtureEvents(texts: readonly string[]): unknown[] {
139 return texts.flatMap(text => unwrapEvents(extractJson(text)))
142/** The issue body for the scheduled run: what left its band, what lacks provenance, and the window. */
143export function driftMarkdown(input: { readonly from: number; readonly to: number; readonly events: number; readonly bands: readonly BandResult[]; readonly problems: readonly Problem[] }): string {
144 const out = [
145 `Re-calibration window ${new Date(input.from).toISOString()} .. ${new Date(input.to).toISOString()} (${input.events} span events).`,
146 '',
147 ]
148 const outside = input.bands.filter(isOutOfBand)
149 if (outside.length > 0) {
150 out.push('## Thresholds out of band', '', ...outside.map(band => `- \`${band.line}\``), '')
151 }
152 if (input.problems.length > 0) {
153 out.push('## Thresholds without provenance', '', ...input.problems.map(problem => `- \`${problem.key}\`: ${problem.problem}`), '')
154 }
155 const rest = input.bands.filter(band => !isOutOfBand(band))
156 if (rest.length > 0) out.push('## Other thresholds with a metric', '', ...rest.map(band => `- \`${band.line}\``), '')
157 out.push('Re-run `node --experimental-strip-types scripts/calibrate/main.ts --since 7d --cite <key>` for a citation block, and update scripts/calibrate/thresholds.json with the new value.', '')
158 return out.join('\n')
161export type Outcome = { readonly code: 0 | 1; readonly text: string; readonly stats: CalibrationStats; readonly bands: readonly BandResult[]; readonly problems: readonly Problem[] }
163/** Everything after the events are in hand. Pure but for reading the registry and the current thresholds. */
164export function evaluate(args: Args, events: readonly unknown[], source: string): Outcome {
165 const stats = calibrationStats(events, { routes: args.routes, attributes: args.attributes, versions: args.versions })
166 const parts = [statsText(stats, args, args.histograms)]
167 const loaded = loadRegistry(args.registry)
168 const query: SpanQuery = {
169 worker: args.worker,
170 from: args.from,
171 to: args.to,
172 sliceMs: args.sliceMs,
173 routes: args.routes,
174 includeNoisy: args.includeNoisy,
175 }
176 for (const key of args.cite) {
177 const entry = loaded.registry.thresholds.get(key)
178 const block = entry && entry.metric ? citationFor({ metric: entry.metric, stats, query: citedQuery(query), from: args.from, to: args.to, source }) : null
179 parts.push(block === null ? `cite ${key}: no metric, or no samples for it in this window\n` : `cite ${key}:\n${JSON.stringify(block, null, 2)}\n`)
180 }
181 if (!args.check) return { code: 0, text: parts.join('\n'), stats, bands: [], problems: [] }
182 const bands = checkBands(loaded.registry, stats)
183 const problems = provenanceProblems(loaded, currentThresholds(args.policy))
184 parts.push(['check:', ...bands.map(band => ` ${band.line}`), ...problems.map(problem => ` provenance ${problem.key}: ${problem.problem}`)].join('\n'))
185 const failed = bands.some(isOutOfBand) || problems.length > 0
186 parts.push(failed ? 'check: FAIL (a threshold left its band, or lacks provenance)\n' : 'check: ok\n')
187 return { code: failed ? 1 : 0, text: parts.join('\n'), stats, bands, problems }
190export async function cli(argv: readonly string[], env: Readonly<Record<string, unknown>>): Promise<number> {
191 let args: Args
192 try {
193 args = parseArgs(argv, Date.now(), env)
194 } catch (error) {
195 process.stderr.write(`calibrate: ${error instanceof Error ? error.message : String(error)}\n`)
196 return 2
197 }
198 let events: unknown[]
199 let source: string
200 try {
201 if (args.fixtures.length > 0) {
202 events = fixtureEvents(args.fixtures.map(path => readFileSync(path, 'utf8')))
203 source = `fixture ${args.fixtures.join(',')}`
204 } else {
205 const retry = telemetryQueryRetry(loadPolicy(args.policy).policy)
206 events = await querySpanEvents({
207 worker: args.worker,
208 from: args.from,
209 to: args.to,
210 sliceMs: args.sliceMs,
211 routes: args.routes,
212 includeNoisy: args.includeNoisy,
213 }, retry)
214 source = `scripts/calibrate ${new Date(args.to).toISOString()}`
215 }
216 } catch (error) {
217 process.stderr.write(`calibrate: telemetry unavailable: ${error instanceof Error ? error.message : String(error)}\n`)
218 return 2
219 }
220 const outcome = evaluate(args, events, source)
221 process.stdout.write(outcome.text)
222 if (args.json !== null) writeFileSync(args.json, `${JSON.stringify({ from: args.from, to: args.to, worker: args.worker, stats: outcome.stats, bands: outcome.bands, problems: outcome.problems }, null, 2)}\n`)
223 if (args.report !== null && outcome.code !== 0) {
224 writeFileSync(args.report, driftMarkdown({ from: args.from, to: args.to, events: outcome.stats.events, bands: outcome.bands, problems: outcome.problems }))
225 }
226 return outcome.code
229if (import.meta.url === `file://${unwrapOr(atIndex(process.argv, 1), '')}`) {
230 cli(process.argv.slice(2), process.env).then(code => process.exit(code), (error: unknown) => {
231 process.stderr.write(`calibrate: ${error instanceof Error ? (typeof error.stack === 'string' ? error.stack : error.message) : String(error)}\n`)
232 process.exit(2)
233 })
Diagnostics4 git calls · 25 ms
CallStatusSizeTime025 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2004.1 KiB · hit20 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=20
git.response.bytes=4170
git.response.cache=HIT
git.response.duration_ms=20
git.fetch.wants=7062ebfc2bd6971a406461d1a576432bd1fc4f22
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4170
git.pkt.payload.max=4149
git.sideband.band1.bytes=4148
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e308e1cb4815db
Request
503d3e2b…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format