main
1// Every threshold cites its source or says it is a prior (issue #73). Pure: the registry is
2// scripts/calibrate/thresholds.json, the current thresholds are passed in (scripts/calibrate/current.ts reads them from
3// cd.policy.json and the limits modules), and the distributions are scripts/calibrate/stats.ts output.
4//
5// provenanceProblems a threshold with no entry, an entry whose value is stale, a citation missing a field, an entry
6// for a threshold that no longer exists. test/calibrate-provenance.test.ts requires none.
7// checkBands for each entry naming a metric: the observed quantile x headroom against the threshold.
8// tight: the threshold is below the observed quantile (real traffic at that percentile is refused).
9// drift: a calibrated threshold is more than `band` times off what the distribution now suggests.
10// Either fails the scheduled re-calibration (.github/workflows/calibrate.yml).
11import { chainGet, mapGet } from '../../src/absent.ts'
12import type { Option } from '../../src/option.ts'
13import { unwrapOr } from '../../src/option.ts'
14import { asRecord } from '../cd/json.ts'
15import { numericOf, quantileOf, QUANTILES, type CalibrationStats, type Quantile } from './stats.ts'
17export const PRIOR = 'PRIOR (uncalibrated)'
18export const CALIBRATED = 'calibrated'
20/** A citation's required fields: where the distribution came from and how the threshold was set against it. */
21export const CITATION_FIELDS = ['source', 'query', 'window', 'date', 'n', 'percentile', 'headroom'] as const
23export type Metric = {
24 readonly attribute: string
25 /** A `ply.route` value, `api` for every /v1 route, or `*` for every route. */
26 readonly route: string
27 /** 1 is the max. */
28 readonly quantile: Quantile | 1
29 /** The threshold should sit at this multiple of the observed quantile. */
30 readonly headroom: number
31 /** A calibrated threshold more than this factor away from quantile x headroom has drifted. */
32 readonly band: number
33}
35export type Citation = {
36 readonly source: string
37 readonly query: string
38 readonly window: string
39 readonly date: string
40 readonly n: number
41 readonly percentile: string
42 readonly headroom: string
43}
45export type Provenance =
46 | { readonly status: typeof CALIBRATED; readonly value: number | readonly number[]; readonly citation: Citation; readonly metric: Metric | null }
47 | { readonly status: typeof PRIOR; readonly value: number | readonly number[]; readonly why: string; readonly metric: Metric | null }
49export type Registry = {
50 readonly minN: number
51 readonly thresholds: ReadonlyMap<string, Provenance>
52}
54/** A problem with one entry, or with the registry as a whole. */
55export type Problem = { readonly key: string; readonly problem: string }
57const QUANTILE_SET: readonly number[] = [...QUANTILES, 1]
59function text(rec: Record<string, unknown>, key: string): string | null {
60 const value = rec[key]
61 return typeof value === 'string' && value.trim() !== '' ? value : null
62}
64function textOr(rec: Record<string, unknown>, key: string): string {
65 const value = text(rec, key)
66 return value === null ? '' : value
67}
69function valueOf(raw: unknown): number | readonly number[] | null {
70 if (typeof raw === 'number' && Number.isFinite(raw)) return raw
71 if (Array.isArray(raw) && raw.length > 0 && raw.every(item => typeof item === 'number' && Number.isFinite(item))) return raw as number[]
72 return null
73}
75function metricOf(given: Option<unknown>, key: string, problems: Problem[]): Metric | null {
76 if (given.tag === 'none') return null
77 const rec = asRecord(given.value)
78 const bad = (problem: string): null => {
79 problems.push({ key, problem: `metric ${problem}` })
80 return null
81 }
82 if (!rec) return bad('is not an object')
83 const attribute = text(rec, 'attribute')
84 const route = text(rec, 'route')
85 if (!attribute) return bad('has no attribute')
86 if (!route) return bad('has no route')
87 const quantile = rec.quantile
88 if (typeof quantile !== 'number' || !QUANTILE_SET.includes(quantile)) return bad(`quantile must be one of ${QUANTILE_SET.join(', ')}`)
89 const headroom = rec.headroom
90 if (typeof headroom !== 'number' || !(headroom >= 1)) return bad('headroom must be a number >= 1')
91 const band = rec.band
92 if (typeof band !== 'number' || !(band > 1)) return bad('band must be a number > 1')
93 return { attribute, route, quantile: quantile as Quantile | 1, headroom, band }
94}
96function entryOf(key: string, raw: unknown, problems: Problem[]): Provenance | null {
97 const rec = asRecord(raw)
98 if (!rec) {
99 problems.push({ key, problem: 'entry is not an object' })
100 return null
101 }
102 const value = valueOf(rec.value)
103 if (value === null) {
104 problems.push({ key, problem: 'value must be a finite number, or a non-empty array of them' })
105 return null
106 }
107 const metric = metricOf(chainGet(rec, 'metric'), key, problems)
108 if (rec.status === PRIOR) {
109 const why = text(rec, 'why')
110 if (!why) {
111 problems.push({ key, problem: `${PRIOR} needs a why` })
112 return null
113 }
114 return { status: PRIOR, value, why, metric }
115 }
116 if (rec.status === CALIBRATED) {
117 const missing = CITATION_FIELDS.filter(field => field === 'n' ? !(typeof rec.n === 'number' && Number.isInteger(rec.n) && rec.n >= 1) : text(rec, field) === null)
118 if (missing.length > 0) {
119 problems.push({ key, problem: `a calibrated threshold must cite ${missing.join(', ')}` })
120 return null
121 }
122 const date = textOr(rec, 'date')
123 if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) {
124 problems.push({ key, problem: `date ${date} is not YYYY-MM-DD` })
125 return null
126 }
127 const citation: Citation = {
128 source: textOr(rec, 'source'),
129 query: textOr(rec, 'query'),
130 window: textOr(rec, 'window'),
131 date,
132 n: rec.n as number,
133 percentile: textOr(rec, 'percentile'),
134 headroom: textOr(rec, 'headroom'),
135 }
136 return { status: CALIBRATED, value, citation, metric }
137 }
138 problems.push({ key, problem: `status must be "${CALIBRATED}" (with a citation) or "${PRIOR}"; got ${JSON.stringify(rec.status)}` })
139 return null
140}
142/** Parse thresholds.json. Malformed entries are problems, not throws, so one run lists them all. */
143export function readRegistry(raw: unknown): { readonly registry: Registry; readonly problems: readonly Problem[] } {
144 const problems: Problem[] = []
145 const parsedRoot = asRecord(raw)
146 const rec = parsedRoot === null ? {} : parsedRoot
147 const thresholds = new Map<string, Provenance>()
148 const table = asRecord(rec.thresholds)
149 if (!table) problems.push({ key: 'thresholds', problem: 'missing' })
150 for (const [key, entry] of Object.entries(table === null ? {} : table)) {
151 const parsed = entryOf(key, entry, problems)
152 if (parsed) thresholds.set(key, parsed)
153 }
154 // The registry's own floor is a threshold too, and carries its own provenance.
155 const floor = entryOf('minN', rec.minN, problems)
156 const minN = floor !== null && typeof floor.value === 'number' && Number.isInteger(floor.value) && floor.value >= 1 ? floor.value : 1
157 if (floor !== null && minN !== floor.value) problems.push({ key: 'minN', problem: 'value must be an integer >= 1' })
158 return { registry: { minN, thresholds }, problems }
159}
161/** Numeric leaves of a policy JSON, by path, under `policy:`. `calibration` and `schedule` blocks are citations, not thresholds. */
162export function policyThresholds(policy: unknown): Map<string, number> {
163 const out = new Map<string, number>()
164 const walk = (value: unknown, path: string): void => {
165 if (typeof value === 'number') {
166 out.set(`policy:${path}`, value)
167 return
168 }
169 if (Array.isArray(value)) {
170 value.forEach((item, i) => walk(item, `${path}[${i}]`))
171 return
172 }
173 const rec = asRecord(value)
174 if (!rec) return
175 for (const [key, child] of Object.entries(rec)) {
176 if (key === 'calibration' || key === 'schedule') continue
177 walk(child, path === '' ? key : `${path}.${key}`)
178 }
179 }
180 walk(policy, '')
181 return out
182}
184/** Numeric fields of a caps object, by `<source>:<field>`. */
185export function capThresholds(source: string, caps: Readonly<Record<string, unknown>>): Map<string, number> {
186 const out = new Map<string, number>()
187 for (const [key, value] of Object.entries(caps)) if (typeof value === 'number') out.set(`${source}:${key}`, value)
188 return out
189}
191/** `policy:steps[3].dwellMs` → `policy:steps[*].dwellMs`. Null when the key has no index. */
192export function wildcardOf(key: string): string | null {
193 const star = key.replace(/\[\d+\]/g, '[*]')
194 return star === key ? null : star
195}
197function sameValue(want: number | readonly number[], got: number | readonly number[]): boolean {
198 if (typeof want === 'number' || typeof got === 'number') return want === got
199 return want.length === got.length && want.every((value, i) => value === got[i])
200}
202/**
203 * Every current threshold has an entry (exact, or its `[*]` wildcard) whose value matches; every entry names a
204 * threshold that exists. Plus the registry's own parse problems.
205 */
206export function provenanceProblems(parsed: { readonly registry: Registry; readonly problems: readonly Problem[] }, current: ReadonlyMap<string, number>): Problem[] {
207 const problems: Problem[] = [...parsed.problems]
208 const { thresholds } = parsed.registry
209 const used = new Set<string>()
210 const wildValues = new Map<string, number[]>()
211 for (const [key, value] of current) {
212 const exact = thresholds.get(key)
213 if (exact) {
214 used.add(key)
215 if (!sameValue(exact.value, value)) problems.push({ key, problem: `threshold is ${value}, its provenance entry says ${JSON.stringify(exact.value)}: re-cite it or mark it ${PRIOR}` })
216 continue
217 }
218 const star = wildcardOf(key)
219 if (star !== null && thresholds.has(star)) {
220 used.add(star)
221 wildValues.set(star, [...unwrapOr(mapGet(wildValues, star), []), value])
222 continue
223 }
224 problems.push({ key, problem: `threshold ${value} has no provenance: add a citation (${CITATION_FIELDS.join(', ')}) or mark it ${PRIOR}` })
225 }
226 for (const [star, values] of wildValues) {
227 const entry = thresholds.get(star)
228 if (entry && !sameValue(entry.value, values)) problems.push({ key: star, problem: `thresholds are ${JSON.stringify(values)}, the provenance entry says ${JSON.stringify(entry.value)}` })
229 }
230 for (const key of thresholds.keys()) if (!used.has(key)) problems.push({ key, problem: 'provenance entry for a threshold that does not exist' })
231 return problems
232}
234export type BandResult = {
235 readonly key: string
236 readonly status: Provenance['status']
237 readonly value: number
238 readonly metric: Metric
239 readonly verdict: 'ok' | 'tight' | 'drift' | 'insufficient' | 'uncalibrated'
240 readonly n: number
241 readonly observed: number | null
242 /** observed x headroom: what this distribution would set the threshold to. */
243 readonly suggested: number | null
244 readonly line: string
245}
247/** True when a band verdict should fail the scheduled run. */
248export function isOutOfBand(result: BandResult): boolean {
249 return result.verdict === 'tight' || result.verdict === 'drift'
250}
252function fmt(value: number): string {
253 return Number.isInteger(value) ? String(value) : value.toPrecision(4)
254}
256/** Each registry entry with a metric, judged against this window's distributions. */
257export function checkBands(registry: Registry, stats: CalibrationStats): BandResult[] {
258 const out: BandResult[] = []
259 for (const [key, entry] of [...registry.thresholds.entries()].sort((a, b) => (a[0] < b[0] ? -1 : 1))) {
260 const metric = entry.metric
261 if (metric === null || typeof entry.value !== 'number') continue
262 const value = entry.value
263 const label = `${key}=${fmt(value)} ${metric.attribute}@${metric.route} q${metric.quantile}`
264 const dist = numericOf(stats, metric.route, metric.attribute)
265 if (dist === null || dist.n < registry.minN) {
266 const n = dist === null ? 0 : dist.n
267 out.push({ key, status: entry.status, value, metric, verdict: 'insufficient', n, observed: null, suggested: null, line: `insufficient ${label}: n=${n} < ${registry.minN}` })
268 continue
269 }
270 const observed = quantileOf(dist, metric.quantile)
271 const suggested = observed * metric.headroom
272 const seen = `n=${dist.n} observed=${fmt(observed)} suggested=${fmt(suggested)} (x${metric.headroom})`
273 if (value < observed) {
274 out.push({ key, status: entry.status, value, metric, verdict: 'tight', n: dist.n, observed, suggested, line: `tight ${label}: ${seen}; the threshold is below the observed quantile` })
275 continue
276 }
277 if (entry.status === PRIOR) {
278 out.push({ key, status: entry.status, value, metric, verdict: 'uncalibrated', n: dist.n, observed, suggested, line: `uncalibrated ${label}: ${seen}; cite this window to calibrate it` })
279 continue
280 }
281 const low = suggested / metric.band
282 const high = suggested * metric.band
283 if (suggested > 0 && (value < low || value > high)) {
284 out.push({ key, status: entry.status, value, metric, verdict: 'drift', n: dist.n, observed, suggested, line: `drift ${label}: ${seen}; outside ${fmt(low)}..${fmt(high)}` })
285 continue
286 }
287 out.push({ key, status: entry.status, value, metric, verdict: 'ok', n: dist.n, observed, suggested, line: `ok ${label}: ${seen}` })
288 }
289 return out
290}
292/** A citation block for one metric over this window, ready to paste into thresholds.json. */
293export function citationFor(input: {
294 readonly metric: Metric
295 readonly stats: CalibrationStats
296 readonly query: unknown
297 readonly from: number
298 readonly to: number
299 readonly source: string
300}): Record<string, unknown> | null {
301 const dist = numericOf(input.stats, input.metric.route, input.metric.attribute)
302 if (dist === null) return null
303 const observed = quantileOf(dist, input.metric.quantile)
304 return {
305 status: CALIBRATED,
306 value: observed * input.metric.headroom,
307 source: input.source,
308 query: JSON.stringify(input.query),
309 window: `${new Date(input.from).toISOString()}..${new Date(input.to).toISOString()}`,
310 date: new Date(input.to).toISOString().slice(0, 10),
311 n: dist.n,
312 percentile: `${input.metric.attribute} @ ${input.metric.route} q${input.metric.quantile} = ${observed}`,
313 headroom: `x${input.metric.headroom}`,
314 metric: input.metric,
315 }
316}
Diagnostics4 git calls · 27 ms
CallStatusSizeTime027 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit5 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=5 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=5 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2004.3 KiB · hit19 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=19 git.response.bytes=4362 git.response.cache=HIT git.response.duration_ms=19 git.fetch.wants=1094f1c5e6fc5844075c8000e6014bf263084452 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4362 git.pkt.payload.max=4341 git.sideband.band1.bytes=4340 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e34c5858888a31
- Request
- b58534d6…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format