main
ply4 git calls331 ms46 KiB2 cachedprotocol v2
263 lines14 KiBRaw
1// Telemetry gate for a canary: reads Workers Observability for one Worker version and proves that
2// version answered every one of the smoke's requests, with no exception and no 5xx.
3//
4// env -u CLOUDFLARE_API_TOKEN node scripts/canary-o11y.mjs <version-id> <from-ms> <ids.txt> [--wait 60]
5//
6// <ids.txt>: the smoke requests, one JSON object per line: { id: x-ply-request-id, version: the x-ply-version the smoke
7// saw on that response, step } (written by `npm run cd` from the smoke JSON). Each id should appear in a
8// `ply.request` log line emitted by <version-id> (the request id is minted per request in src/index.js, so a match proves
9// which version ran it).
10//
11// Span events have no script version. Traces are queried by service only, then fetch spans (name exactly `fetch`) are
12// kept when the trace contains a `$metadata.requestId` from this version's events. Fetch p50/p99 are on the JSON summary.
13//
14// Every query is narrow, so other traffic on the Worker can never crowd smoke events out of a result:
15// - smoke logs: `id in (<≤50 smoke ids>)` AND scriptVersion = <version>, one query per chunk of ids;
16// - smoke invocations: `$metadata.requestId in (...)` of the requests those logs belong to (outcome, wall, cpu);
17// - health: only the bad events of <version> since <from-ms>: fetch invocations with outcome != ok, ply.request logs
18// with status >= 500, and level=error events.
19// Each query pages with the `offset` cursor until a page comes back short, so no result is ever truncated at `limit`.
20// (Before 2026-10-31 this was one limit-1000 query over all of the version's events. Deploy b9a40b21: ~560 foreign
21// invocations in the window pushed 23 of 79 smoke ids past the cap and failed a healthy deploy.)
22//
23// Ingest lag: a missing id is re-queried with backoff (5, 10, 15, 15 … s) until --wait seconds (default 60) have passed
24// since the first query. Lag observed is 10-30 s.
25// Loss: Workers Observability sometimes never ingests an event. Deploy b9a40b21 (2026-10-31 18:25Z): 5 of 79 smoke
26// requests (all answered 2xx/4xx by b9a40b21, per x-ply-version) had no ply.request log 15 min later; 4 of them had no
27// invocation event either, and 3 no otel root span, while their neighbours a few ms apart were ingested within seconds.
28// Across 2026-10-31 ≈12 of ~830 smoke requests (~1.4%, bursty) were lost on healthy versions. So after --wait, each
29// still-missing id is looked up under every version, and scripts/canary-verdict.mjs decides: an id logged by another
30// version fails (routing); an id logged nowhere is a tolerated loss iff the smoke recorded x-ply-version: <version> on its
31// response and at most max(2, 2%) of the phase's ids are lost; otherwise it fails. Each tolerated loss is printed (id,
32// step) on stderr, and stderr's last line is the verdict, e.g. `telemetry: 117/119 matched, 2 lost (tolerated)`.
33//
34// Fails (exit 1) when the verdict fails, i.e. also when any fetch invocation of <version> since <from-ms> has an outcome
35// other than ok, when any of its ply.request logs has a 5xx status, or when it logged at level error.
36// Prints one JSON summary on stdout (with `telemetry`, the verdict line).
37import { execFileSync } from 'node:child_process';
38import { readFileSync } from 'node:fs';
39import { fileURLToPath } from 'node:url';
40import { atIndex, chainGet, dig, recordGet } from '../src/absent.ts';
41import { unwrapOr } from '../src/option.ts';
42import { parseRequests, verdict } from './canary-verdict.mjs';
43import { dedupeRows, pageCursor } from './cd/telemetry.ts';
45function asRecord(value) {
46 return value !== null && typeof value === 'object' && !Array.isArray(value) ? value : null;
49function own(record, key) {
50 const host = asRecord(record);
51 if (host === null) return { tag: 'none' };
52 return recordGet(host, key);
55function recordOrEmpty(first, second) {
56 if (first !== null) return first;
57 if (second !== null) return second;
58 return {};
61function present(option) {
62 return option.tag === 'some' ? option.value : null;
65// cf creates node_modules/.cache under cwd, so every query runs from the repo root, never the caller's directory.
66const ROOT = fileURLToPath(new URL('..', import.meta.url));
68const [version, fromArg, idsFile] = process.argv.slice(2);
69const arg = k => {
70 const i = process.argv.indexOf(`--${k}`);
71 if (i <= 0) return null;
72 return unwrapOr(atIndex(process.argv, i + 1), null);
73};
74if (!version || !fromArg || !idsFile) { console.error('usage: canary-o11y.mjs <version-id> <from-ms> <ids.txt> [--wait s]'); process.exit(2); }
75const requests = parseRequests(readFileSync(idsFile, 'utf8'));
76const want = [...new Set(requests.map(r => r.id))];
77const waitArg = arg('wait');
78const waitS = Number(waitArg === null ? 60 : waitArg);
79const from = Math.floor(Number(fromArg));
80const PAGE = 1000, CHUNK = 50, MAX_PAGES = 50;
82const f = (key, operation, value, type = 'string') => ({ key, operation, type, value });
83const SERVICE = f('$metadata.service', 'eq', 'ply');
84const VERSION = f('$workers.scriptVersion.id', 'eq', version);
85const chunks = xs => Array.from({ length: Math.ceil(xs.length / CHUNK) }, (_, i) => xs.slice(i * CHUNK, (i + 1) * CHUNK));
87function cfQuery(body) {
88 return JSON.parse(execFileSync('npx', ['cf', 'o11y', 'telemetry', 'query', '--body', JSON.stringify(body)], {
89 encoding: 'utf8', maxBuffer: 64 << 20, cwd: ROOT,
90 }));
93function pageOf(res, view) {
94 if (view === 'events') {
95 const events = chainGet(res, 'events');
96 const rows = events.tag === 'some' ? chainGet(events.value, 'events') : { tag: 'none' };
97 return rows.tag === 'some' ? rows.value : [];
98 }
99 const traces = chainGet(res, 'traces');
100 if (traces.tag === 'none') return [];
101 const nested = chainGet(traces.value, 'traces');
102 if (nested.tag === 'some') return nested.value;
103 const events = chainGet(traces.value, 'events');
104 return events.tag === 'some' ? events.value : [];
107// Every page matching `filters` in [from, now + 60 s].
108function queryView(view, filters) {
109 const out = [];
110 let offset;
111 for (let page = 0; page < MAX_PAGES; page++) {
112 const body = {
113 queryId: 'ply-canary', timeframe: { from, to: Date.now() + 60_000 }, view, limit: PAGE,
114 ...(offset ? { offset } : {}), parameters: { filters },
115 };
116 const rows = pageOf(cfQuery(body), view);
117 out.push(...rows);
118 if (rows.length < PAGE) return dedupeRows(out);
119 offset = pageCursor(rows);
120 if (!offset) throw new Error(`a full page with no $metadata.id cursor (${view})`);
121 }
122 throw new Error(`more than ${MAX_PAGES} pages of ${PAGE} ${view} for ${JSON.stringify(filters)}`);
125const events = filters => queryView('events', filters);
126const smokeLogs = (ids, filters) => chunks(ids).flatMap(c => events([SERVICE, ...filters, f('msg', 'eq', 'ply.request'), f('id', 'in', c.join(','))]));
128const t0 = Date.now();
129const logs = new Map(); // smoke id → its ply.request log events from <version>
130const sleep = ms => new Promise(r => setTimeout(r, ms));
131for (let attempt = 0; ; attempt++) {
132 const missing = want.filter(id => !logs.has(id));
133 for (const e of smokeLogs(missing, [VERSION])) {
134 const id = present(own(present(chainGet(e, 'source')), 'id'));
135 if (!logs.has(id)) logs.set(id, []);
136 logs.get(id).push(e);
137 }
138 const elapsed = Date.now() - t0;
139 if (logs.size === want.length || elapsed >= waitS * 1000) break;
140 const pauses = [5, 10, 15];
141 await sleep(Math.min(unwrapOr(atIndex(pauses, attempt), 15), waitS) * 1000);
143const missing = want.filter(id => !logs.has(id));
144const loggedBy = new Map([...logs.keys()].map(id => [id, new Set([version])]));
145for (const e of missing.length ? smokeLogs(missing, []) : []) {
146 const id = present(own(present(chainGet(e, 'source')), 'id'));
147 const versionId = dig(e, ['$workers', 'scriptVersion', 'id']);
148 const v = versionId.tag === 'none' ? 'unknown' : versionId.value;
149 if (!loggedBy.has(id)) loggedBy.set(id, new Set());
150 loggedBy.get(id).add(v);
152function hasStatus(event) {
153 return own(present(chainGet(event, 'source')), 'status').tag === 'some';
155const smokeLog = [...logs.values()].map(events => {
156 for (const event of events) if (hasStatus(event)) return event;
157 return unwrapOr(atIndex(events, 0), null);
158});
159const requestIds = [...new Set(smokeLog.map(event => present(own(present(chainGet(event, '$metadata')), 'requestId'))).filter(Boolean))];
160const inv = chunks(requestIds).flatMap(c => events([SERVICE, VERSION, f('$metadata.type', 'eq', 'cf-worker-event'), f('$metadata.requestId', 'in', c.join(','))]));
161const nonOk = events([SERVICE, VERSION, f('$metadata.type', 'eq', 'cf-worker-event'), f('$workers.eventType', 'eq', 'fetch'), f('$workers.outcome', 'neq', 'ok')]);
162const fivexx = events([SERVICE, VERSION, f('msg', 'eq', 'ply.request'), f('status', 'gte', 500, 'number')]);
163const errors = events([SERVICE, VERSION, f('$metadata.level', 'eq', 'error')]);
165const count = (xs, read) => {
166 const out = {};
167 for (const item of xs) {
168 const got = read(item);
169 const key = got.tag === 'none' ? 'un' + 'defined' : String(got.value);
170 const next = out[key];
171 out[key] = typeof next === 'number' ? next + 1 : 1;
172 }
173 return out;
174};
175const pct = (xs, p) => { const s = xs.filter(x => Number.isFinite(x)).sort((a, b) => a - b); return s.length ? s[Math.floor((s.length - 1) * p)] : null; };
176const workerNumber = (event, key) => {
177 const value = own(present(chainGet(event, '$workers')), key);
178 return value.tag === 'some' ? value.value : null;
179};
180const wall = inv.map(event => workerNumber(event, 'wallTimeMs')), cpu = inv.map(event => workerNumber(event, 'cpuTimeMs'));
182// Fetch spans carry no script version. Query traces by service, then keep a span named `fetch` only when its trace
183// contains a request id from this version's events.
184function spanDurationMs(span) {
185 const attrs = recordOrEmpty(asRecord(span.attributes), asRecord(span.attrs));
186 for (const candidate of [span.durationMs, span.duration_ms, attrs.duration_ms, attrs.durationMs]) {
187 if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate >= 0) return candidate;
188 }
189 const raw = typeof span.duration === 'number' ? span.duration : attrs.duration;
190 if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) return null;
191 return raw > 1_000_000 ? raw / 1_000_000 : raw;
193function walk(node, visit, seen) {
194 if (node === null || typeof node !== 'object' || seen.has(node)) return;
195 seen.add(node);
196 if (Array.isArray(node)) { for (const item of node) walk(item, visit, seen); return; }
197 visit(node);
198 for (const value of Object.values(node)) walk(value, visit, seen);
200const versionRequestIds = new Set();
201for (const e of [...smokeLog, ...inv, ...nonOk, ...fivexx, ...errors]) {
202 const id = present(own(present(chainGet(e, '$metadata')), 'requestId'));
203 if (typeof id === 'string' && id) versionRequestIds.add(id);
205const fetchMs = [];
206for (const trace of versionRequestIds.size ? queryView('traces', [SERVICE]) : []) {
207 const strings = new Set();
208 walk(trace, rec => { for (const value of Object.values(rec)) if (typeof value === 'string') strings.add(value); }, new Set());
209 let hit = false;
210 for (const id of versionRequestIds) if (strings.has(id)) { hit = true; break; }
211 if (!hit) continue;
212 walk(trace, rec => {
213 if (rec.name !== 'fetch') return;
214 const duration = spanDurationMs(rec);
215 if (duration !== null) fetchMs.push(duration);
216 }, new Set());
218function kept(option) {
219 return option.tag === 'some' ? { keep: true, value: option.value } : { keep: false };
221function firstText(first, second) {
222 if (first.tag === 'some') return String(first.value);
223 if (second.tag === 'some') return String(second.value);
224 return '';
226const brief = event => {
227 const viaWorkers = dig(event, ['$workers', 'event', 'request', 'path']);
228 const viaSource = own(present(chainGet(event, 'source')), 'path');
229 const path = viaWorkers.tag === 'some' ? kept(viaWorkers) : kept(viaSource);
230 const outcome = kept(own(present(chainGet(event, '$workers')), 'outcome'));
231 const status = kept(own(present(chainGet(event, 'source')), 'status'));
232 const message = firstText(
233 chainGet(present(chainGet(event, 'source')), 'message'),
234 chainGet(present(chainGet(event, '$metadata')), 'message'),
235 );
236 const row = { t: new Date(event.timestamp).toISOString() };
237 if (path.keep) row.path = path.value;
238 if (outcome.keep) row.outcome = outcome.value;
239 if (status.keep) row.status = status.value;
240 row.message = message.slice(0, 200);
241 return row;
242};
243const health = { nonOk: nonOk.length, fivexx: fivexx.length, errorLogs: errors.length };
244const v = verdict({ version, requests, loggedBy, health });
245for (const r of v.lost) console.error(`telemetry loss${v.lost.length <= v.allowance ? ' (tolerated)' : ''}: ${r.id} ${r.step} (answered by ${r.version}; logged by no version)`);
246for (const r of v.unconfirmed) console.error(`telemetry loss (unconfirmed): ${r.id} ${r.step} (smoke saw x-ply-version ${r.version}; logged by no version)`);
247for (const r of v.otherVersion) console.error(`other version: ${r.id} ${r.step} logged by ${r.loggedBy.join(',')}`);
248console.error(v.line);
249const summary = {
250 telemetry: v.line, pass: v.pass, reasons: v.reasons,
251 version, from: new Date(from).toISOString(), waitedS: Math.round((Date.now() - t0) / 1000),
252 smokeRequests: want.length, matched: v.matched, missing, allowance: v.allowance,
253 lost: v.lost, unconfirmed: v.unconfirmed, otherVersion: v.otherVersion,
254 smokeInvocations: inv.length,
255 smokeOutcomes: count(inv, event => own(present(chainGet(event, '$workers')), 'outcome')),
256 smokeStatuses: count(smokeLog, event => own(present(chainGet(event, 'source')), 'status')),
257 ...health,
258 bad: [...nonOk, ...fivexx, ...errors].slice(0, 20).map(brief),
259 wallMs: { p50: pct(wall, 0.5), p95: pct(wall, 0.95) }, cpuMs: { p50: pct(cpu, 0.5), p95: pct(cpu, 0.95) },
260 fetchMs: { p50: pct(fetchMs, 0.5), p99: pct(fetchMs, 0.99) },
261};
262console.log(JSON.stringify(summary));
263process.exit(v.pass ? 0 : 1);
Diagnostics4 git calls · 331 ms
CallStatusSizeTime0331 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=4
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=4
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2005.1 KiB324 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=324
git.response.bytes=5176
git.response.cache=MISS
git.response.duration_ms=324
git.fetch.wants=60a5405ada792cd3b621b961f86d05fd51143d6a
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5176
git.pkt.payload.max=5155
git.sideband.band1.bytes=5154
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e35dd29ea65cd4
Request
639a5c32…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format