main
1// p50/p95/p99 of flags.eval by cache status, and the count of resolved variants by flag and revision.
2// Unit id is not a dimension. The same percentile index as scripts/cd/telemetry.ts: floor((n - 1) * p).
3import { atIndex, chainGet, mapGet } from '../absent.ts';
4import { unwrapOr } from '../option.ts';
6export type Quantiles = { readonly n: number; readonly p50: number | null; readonly p95: number | null; readonly p99: number | null };
8export type CacheRow = { readonly cacheStatus: string } & Quantiles;
10export type VariantRow = {
11 readonly key: string;
12 readonly revision: string;
13 readonly variantKind: string;
14 readonly variantValue: string;
15 readonly reason: string;
16 readonly count: number;
17};
19export type FlagReport = {
20 readonly cache: readonly CacheRow[];
21 readonly variants: readonly VariantRow[];
22};
24type Obj = Record<string, unknown>;
26function record(value: unknown): Obj | null {
27 return value && typeof value === 'object' && !Array.isArray(value) ? value as Obj : null;
28}
30function stringField(value: unknown): string | null {
31 return typeof value === 'string' && value.length > 0 ? value : null;
32}
34function field(value: unknown, key: string): unknown {
35 return unwrapOr(chainGet(value, key), null);
36}
38function recordOrEmpty(first: Obj | null, second: Obj | null): Obj {
39 if (first !== null) return first;
40 if (second !== null) return second;
41 return {};
42}
44function firstString(...values: readonly unknown[]): string | null {
45 for (const value of values) {
46 const text = stringField(value);
47 if (text !== null) return text;
48 }
49 return null;
50}
52function firstPositive(...values: readonly unknown[]): number | null {
53 for (const value of values) {
54 const ms = positiveMs(value);
55 if (ms !== null) return ms;
56 }
57 return null;
58}
60function fieldsOf(event: Obj): Obj {
61 const source = recordOrEmpty(record(event.source), null);
62 const attributes = recordOrEmpty(record(event.attributes), record(field(source, 'attributes')));
63 return { ...attributes, ...source, ...event };
64}
66export function percentile(values: readonly number[], p: number): number | null {
67 const sorted = values.filter(value => Number.isFinite(value)).sort((a, b) => a - b);
68 if (sorted.length === 0) return null;
69 return unwrapOr(atIndex(sorted, Math.floor((sorted.length - 1) * p)), null);
70}
72function quantiles(values: readonly number[]): Quantiles {
73 return { n: values.length, p50: percentile(values, 0.5), p95: percentile(values, 0.95), p99: percentile(values, 0.99) };
74}
76type EvalPoint = { readonly cacheStatus: string; readonly resolveMs: number };
78/** A positive duration. Zero is not a measurement: the Workers clock did not move. */
79function positiveMs(value: unknown): number | null {
80 return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : null;
81}
83function wallMs(fields: Obj): number | null {
84 const workers = record(fields.$workers);
85 return firstPositive(field(workers, 'wallTimeMs'), fields.wallTimeMs, fields['$workers.wallTimeMs']);
86}
88function spanMs(fields: Obj): number | null {
89 return firstPositive(fields.durationMs, fields.durationMS, fields.duration_ms);
90}
92/**
93 * Prefer the awaited cache/R2 wall time. A legacy positive eval_ms is the next choice.
94 * Zero is skipped so a frozen Workers clock does not become the percentile: the invocation
95 * wallTimeMs, then the span duration, fill that gap.
96 */
97function measuredMs(fields: Obj): number | null {
98 return firstPositive(
99 fields.resolve_ms,
100 fields['flags.resolve_ms'],
101 fields.eval_ms,
102 fields['flags.eval_ms'],
103 wallMs(fields),
104 spanMs(fields),
105 );
106}
107type ResolvePoint = { readonly key: string; readonly revision: string; readonly variantKind: string; readonly variantValue: string; readonly reason: string };
109function evalPoint(event: Obj): EvalPoint | null {
110 const fields = fieldsOf(event);
111 const name = firstString(fields.name, fields.spanName);
112 const msg = stringField(fields.msg);
113 if (msg !== 'flags.eval' && name !== 'flags.eval') return null;
114 const resolveMs = measuredMs(fields);
115 const cacheStatus = firstString(fields.cache_status, fields['flags.cache_status']);
116 if (resolveMs === null || !cacheStatus) return null;
117 return { cacheStatus, resolveMs };
118}
120function resolvePoint(event: Obj): ResolvePoint | null {
121 const fields = fieldsOf(event);
122 const name = firstString(fields.name, fields.spanName);
123 const msg = stringField(fields.msg);
124 if (msg !== 'flags.resolve' && name !== 'flags.resolve') return null;
125 const key = firstString(fields.key, fields['flags.key']);
126 const revisionText = firstString(fields.revision, fields['flags.revision']);
127 const revision = revisionText === null ? '' : revisionText;
128 const variantKind = firstString(fields.variant_kind, fields['flags.variant_kind']);
129 const variantValue = firstString(fields.variant_value, fields['flags.variant_value']);
130 const reason = firstString(fields.reason, fields['flags.reason']);
131 if (!key || !variantKind || variantValue === null || !reason) return null;
132 return { key, revision, variantKind, variantValue, reason };
133}
135export function reportFromEvents(events: readonly unknown[]): FlagReport {
136 const durations = new Map<string, number[]>();
137 const variants = new Map<string, VariantRow>();
138 for (const event of events) {
139 const rec = record(event);
140 if (!rec) continue;
141 const sample = evalPoint(rec);
142 if (sample) {
143 const list = unwrapOr(mapGet(durations, sample.cacheStatus), []);
144 list.push(sample.resolveMs);
145 durations.set(sample.cacheStatus, list);
146 }
147 const resolved = resolvePoint(rec);
148 if (!resolved) continue;
149 const id = [resolved.key, resolved.revision, resolved.variantKind, resolved.variantValue, resolved.reason].join('\0');
150 const row = variants.get(id);
151 if (row) variants.set(id, { ...row, count: row.count + 1 });
152 else variants.set(id, { ...resolved, count: 1 });
153 }
154 const cache = [...durations.entries()]
155 .sort((a, b) => a[0].localeCompare(b[0]))
156 .map(([cacheStatus, values]) => ({ cacheStatus, ...quantiles(values) }));
157 const rows = [...variants.values()].sort((a, b) => a.key.localeCompare(b.key) || a.revision.localeCompare(b.revision) || b.count - a.count || a.reason.localeCompare(b.reason));
158 return { cache, variants: rows };
159}
161/**
162 * What the percentile columns measure. stdout of scripts/flags-o11y.ts stays the table;
163 * the script prints this on stderr.
164 */
165export const MEASUREMENT =
166 'p50/p95/p99 are resolve_ms on flags.eval: the awaited FlagsCache read (a hit is the cache lookup; a miss, stale, or revalidation includes the R2 read inside that await). Grouped by cache_status. Zero is not a sample. Fallback is a positive eval_ms, then invocation wallTimeMs, then span duration. flags.resolve events are variant counts only and are not timed again.';
168export function renderReport(report: FlagReport): string {
169 const lines = ['cache_status n p50_ms p95_ms p99_ms'];
170 for (const row of report.cache) lines.push([row.cacheStatus, row.n, quantileCell(row.p50), quantileCell(row.p95), quantileCell(row.p99)].join(' '));
171 lines.push('', 'flag revision variant_kind variant_value reason count');
172 for (const row of report.variants) {
173 lines.push([row.key, row.revision || '-', row.variantKind, row.variantValue, row.reason, row.count].join(' '));
174 }
175 return lines.join('\n');
176}
178export type TelemetryFilter = { readonly key: string; readonly operation: string; readonly type: string; readonly value: string };
180/** Workers observability query for one low-cardinality log. The filter set has no unit id. */
181function quantileCell(value: number | null): number | string {
182 return value === null ? '-' : value;
183}
185export function flagLogQuery(worker: string, msg: 'flags.eval' | 'flags.resolve', from: number, to: number, offset: string | null) {
186 const filters: TelemetryFilter[] = [
187 { key: '$metadata.service', operation: 'eq', type: 'string', value: worker },
188 { key: 'msg', operation: 'eq', type: 'string', value: msg },
189 ];
190 return {
191 queryId: 'ply-flags',
192 timeframe: { from, to },
193 view: 'events' as const,
194 limit: 1000,
195 ...(offset ? { offset } : {}),
196 parameters: { filters },
197 };
198}
Diagnostics4 git calls · 30 ms
CallStatusSizeTime030 ms
discover200152 B4 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=4 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2002.6 KiB · hit24 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=24 git.response.bytes=2632 git.response.cache=HIT git.response.duration_ms=24 git.fetch.wants=77e24c68a85571b70d196b6a6aef43449ec2c344 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2632 git.pkt.payload.max=2611 git.sideband.band1.bytes=2610 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2eb0e0cf34bc2
- Request
- 328936c0…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format