main
1import assert from 'node:assert/strict'
2import { dig } from '../src/absent.ts'
3import { unwrapOr } from '../src/option.ts'
4import { readFileSync } from 'node:fs'
5import { test } from 'node:test'
6import { JOIN_MIN_LOGGED, SMOKE_LATENCY } from '../scripts/cd/baseline.ts'
7import { artifactsProbeUrl, revokeConfirmed } from '../scripts/cd/classify.ts'
8import { raysByRequestId } from '../scripts/cd/csv.ts'
9import { asRecord } from '../scripts/cd/json.ts'
10import { DEFAULT_MAX_LOSS_RATE, DEFAULT_POLICY_PATH, loadPolicy } from '../scripts/cd/policy.ts'
11import { ALL_INVOCATIONS, progressLine } from '../scripts/cd/ramp.ts'
12import { readSmokeRequests } from '../scripts/cd/stage-telemetry.ts'
13import { judgeStep, routedShare } from '../scripts/cd/slo.ts'
14import {
15 DEFAULT_ENTRYPOINT,
16 fetchInvocation,
17 fetchSample,
18 isTraceSummary,
19 judgeSmokeTelemetry,
20 implausibleJoin,
21 pageCursor,
22 percentile,
23 queryFault,
24 statsFromEvents,
25 summarizeTelemetry,
26 telemetryLoss,
27 unwrapEvents,
28 type SmokeRequest,
29} from '../scripts/cd/telemetry.ts'
31const VERSION = '8c1af0e2-7b3d-4c1a-9e20-6f0a1b2c3d4e'
32const SMOKE_ID = '11111111-1111-4111-8111-111111111111'
33const PLATFORM = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
34const RAY = '8f3a2b1c4d5e6f70-IAD'
35const OTHER_RAY = 'ffffffffffffffff-IAD'
36const LOG_TRACE = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
37const SPAN_TRACE = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
39/** A Workers Observability span event: requestId is null; traceId and rayId are set. */
40function spanEvent(name: string, durationMs: number, rayId: string | null, traceId: string) {
41 return {
42 $metadata: {
43 id: `span-${name}-${traceId.slice(0, 8)}`,
44 requestId: null,
45 traceId,
46 service: 'ply',
47 type: 'span',
48 },
49 traceId,
50 rayId,
51 name,
52 durationMs,
53 }
54}
56function smoke(id: string, ray: string | null, status = 200): SmokeRequest {
57 return { id, version: VERSION, step: 'commit', status, expected: [200], ray }
58}
60function logEvent(id: string, requestId: string, traceId: string, status = 200) {
61 return {
62 $metadata: { id: `log-${id}`, requestId, traceId, service: 'ply', level: 'info' },
63 $workers: { scriptVersion: { id: VERSION }, outcome: 'ok' },
64 source: { msg: 'ply.request', id, status, method: 'POST', path: '/v1/commit', ms: 80 },
65 }
66}
68function invocation(id: string, requestId: string, traceId: string) {
69 return {
70 $metadata: { id: `inv-${id}`, requestId, traceId, type: 'cf-worker-event', service: 'ply' },
71 $workers: { scriptVersion: { id: VERSION }, eventType: 'fetch', outcome: 'ok', wallTimeMs: 80, cpuTimeMs: 12 },
72 }
73}
75test('span events join on rayId when requestId is null', () => {
76 const fetch = spanEvent('fetch', 20, RAY, SPAN_TRACE)
77 const upload = spanEvent('git.upload-pack fetch', 30, RAY, SPAN_TRACE)
78 const receive = spanEvent('git.receive-pack', 40, RAY, SPAN_TRACE)
79 const spans = [fetch, upload, receive]
80 const encoded = JSON.stringify(spans)
81 assert.equal(fetch.$metadata.requestId, null)
82 assert.equal(encoded.includes(PLATFORM), false)
83 assert.equal(encoded.includes(SMOKE_ID), false)
84 const summary = summarizeTelemetry(VERSION, [smoke(SMOKE_ID, RAY)], [logEvent(SMOKE_ID, PLATFORM, LOG_TRACE), invocation(SMOKE_ID, PLATFORM, LOG_TRACE)], spans)
85 assert.equal(summary.tracedIds.has(SMOKE_ID), true)
86 assert.deepEqual(summary.fetchMs, [20])
87 const judge = judgeSmokeTelemetry({ version: VERSION, requests: [smoke(SMOKE_ID, RAY)], summary, ceilings: SMOKE_LATENCY, maxLossRate: null })
88 assert.equal(judge.kind, 'pass')
89})
91test('a colo-less rayId still joins the cf-ray from the smoke CSV', () => {
92 const fetch = spanEvent('fetch', 20, '8f3a2b1c4d5e6f70', SPAN_TRACE)
93 const summary = summarizeTelemetry(VERSION, [smoke(SMOKE_ID, RAY)], [logEvent(SMOKE_ID, PLATFORM, LOG_TRACE), invocation(SMOKE_ID, PLATFORM, LOG_TRACE)], [fetch])
94 assert.equal(summary.tracedIds.has(SMOKE_ID), true)
95 assert.deepEqual(summary.fetchMs, [20])
96})
98test('traceId joins a span when rayId does not', () => {
99 const fetch = spanEvent('fetch', 15, null, LOG_TRACE)
100 assert.equal(fetch.$metadata.requestId, null)
101 assert.equal(JSON.stringify(fetch).includes(SMOKE_ID), false)
102 assert.equal(JSON.stringify(fetch).includes(RAY), false)
103 const summary = summarizeTelemetry(VERSION, [smoke(SMOKE_ID, RAY)], [logEvent(SMOKE_ID, PLATFORM, LOG_TRACE), invocation(SMOKE_ID, PLATFORM, LOG_TRACE)], [fetch])
104 assert.equal(summary.tracedIds.has(SMOKE_ID), true)
105 assert.deepEqual(summary.fetchMs, [15])
106})
108test('a join that misses the invocations the worker logged is a harness fault', () => {
109 const requests: SmokeRequest[] = []
110 const events: unknown[] = []
111 const traces: unknown[] = []
112 for (let i = 0; i < JOIN_MIN_LOGGED; i++) {
113 const id = `22222222-2222-4222-8222-22222222222${i}`
114 const requestId = `bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb${i}`
115 const ray = `aaaaaaaaaaaaaaa${i}-IAD`
116 requests.push(smoke(id, ray))
117 events.push(logEvent(id, requestId, LOG_TRACE.replace(/a/g, String(i))), invocation(id, requestId, LOG_TRACE.replace(/a/g, String(i))))
118 traces.push(spanEvent('fetch', 20, OTHER_RAY, SPAN_TRACE), spanEvent('git.upload-pack fetch', 30, OTHER_RAY, SPAN_TRACE), spanEvent('git.receive-pack', 40, OTHER_RAY, SPAN_TRACE))
119 }
120 const summary = summarizeTelemetry(VERSION, requests, events, traces)
121 assert.equal(summary.tracedIds.size, 0)
122 assert.equal(summary.invocations, JOIN_MIN_LOGGED)
123 assert.equal(implausibleJoin({ smoke: requests.length, invocations: summary.invocations, joined: summary.tracedIds.size }), true)
124 const judge = judgeSmokeTelemetry({ version: VERSION, requests, summary, ceilings: SMOKE_LATENCY, maxLossRate: null })
125 assert.equal(judge.kind, 'harness_fault')
126 if (judge.kind === 'harness_fault') assert.match(judge.reason, /span join matched 0\//)
127})
129test('a 5xx stays a worker fault when the span join also misses', () => {
130 const requests: SmokeRequest[] = []
131 const events: unknown[] = []
132 const traces: unknown[] = []
133 for (let i = 0; i < JOIN_MIN_LOGGED; i++) {
134 const id = `33333333-3333-4333-8333-33333333333${i}`
135 const requestId = `cccccccc-cccc-4ccc-8ccc-ccccccccccc${i}`
136 const ray = `bbbbbbbbbbbbbbb${i}-IAD`
137 requests.push({ ...smoke(id, ray), status: 500 })
138 events.push(logEvent(id, requestId, SPAN_TRACE, 500), invocation(id, requestId, SPAN_TRACE))
139 traces.push(spanEvent('fetch', 20, OTHER_RAY, LOG_TRACE))
140 }
141 const summary = summarizeTelemetry(VERSION, requests, events, traces)
142 const judge = judgeSmokeTelemetry({ version: VERSION, requests, summary, ceilings: SMOKE_LATENCY, maxLossRate: null })
143 assert.equal(judge.kind, 'worker')
144})
146test('a telemetry query failure is a harness fault and not a worker fault', () => {
147 const fault = queryFault('telemetry unavailable: cf o11y down')
148 assert.equal(fault.kind, 'harness_fault')
149 assert.match(fault.reason, /cf o11y down/)
150})
152test('the smoke CSV cf-ray is the join key the report can omit', () => {
153 const csv = [
154 'step,method,path,status,ms,pass,cf_ray,x_ply_request_id,x_artifacts_ray,x_ply_version,x_ply_build,server_timing,detail',
155 `commit,POST,/v1/commit,200,10,pass,${RAY},${SMOKE_ID},art-ray,${VERSION},build,total;dur=10,ok`,
156 ].join('\n')
157 assert.equal(raysByRequestId(csv).get(SMOKE_ID), RAY)
158 const requests = readSmokeRequests(JSON.stringify({
159 requests: [{ id: SMOKE_ID, version: VERSION, step: 'commit', status: 200, expected: [200] }],
160 }))
161 assert.equal(requests[0]?.ray, null)
162 const ray = raysByRequestId(csv).get(SMOKE_ID) ?? null
163 const joined = requests.map(request => request.ray ? request : { ...request, ray })
164 assert.equal(joined[0]?.ray, RAY)
165})
167test('trace summaries with a full join are a harness fault, not a worker fault', () => {
168 const trace = {
169 traceId: LOG_TRACE,
170 spans: 4,
171 traceDurationMs: 80,
172 rootSpanName: 'POST',
173 rootTransactionName: 'POST https://ply.ivar.workers.dev/v1/commit',
174 errors: [],
175 id: '01TRACE',
176 services: ['ply'],
177 traceStartMs: 1,
178 traceEndMs: 81,
179 }
180 const summary = summarizeTelemetry(
181 VERSION,
182 [smoke(SMOKE_ID, RAY)],
183 [logEvent(SMOKE_ID, PLATFORM, LOG_TRACE), invocation(SMOKE_ID, PLATFORM, LOG_TRACE)],
184 [trace],
185 )
186 assert.equal(isTraceSummary(trace), true)
187 assert.equal(summary.spanShape, 'summaries')
188 assert.equal(summary.tracedIds.has(SMOKE_ID), true)
189 assert.deepEqual([...summary.fetchMs], [])
190 const judge = judgeSmokeTelemetry({ version: VERSION, requests: [smoke(SMOKE_ID, RAY)], summary, ceilings: SMOKE_LATENCY, maxLossRate: null })
191 assert.equal(judge.kind, 'harness_fault')
192 if (judge.kind === 'harness_fault') {
193 assert.match(judge.reason, /trace summaries/)
194 assert.equal(judge.checks.some(check => check.kind === 'shape'), true)
195 }
196})
198test('an observed fetch duration over the ceiling is a worker fault', () => {
199 const slow = {
200 source: {
201 name: 'fetch',
202 durationMS: SMOKE_LATENCY.fetchMs.p99Ms + 1,
203 url: { path: '/git/ply-loadgen/repo.git/git-upload-pack' },
204 http: { response: { status_code: 200 } },
205 },
206 $metadata: { id: 'span-slow', type: 'span', spanName: 'fetch', requestId: null, traceId: LOG_TRACE, rayId: RAY, service: 'ply' },
207 }
208 const sample = fetchSample(slow)
209 assert.equal(sample?.durationMs, SMOKE_LATENCY.fetchMs.p99Ms + 1)
210 assert.equal(sample?.artifacts, true)
211 const summary = summarizeTelemetry(
212 VERSION,
213 [smoke(SMOKE_ID, RAY)],
214 [logEvent(SMOKE_ID, PLATFORM, LOG_TRACE), invocation(SMOKE_ID, PLATFORM, LOG_TRACE)],
215 [slow],
216 )
217 const judge = judgeSmokeTelemetry({ version: VERSION, requests: [smoke(SMOKE_ID, RAY)], summary, ceilings: SMOKE_LATENCY, maxLossRate: null })
218 assert.equal(judge.kind, 'worker')
219 if (judge.kind === 'worker') assert.equal(judge.failures.some(failure => failure.kind === 'latency'), true)
220})
222const CANARY = 'c2713bac-95bb-4a4a-b3c1-f5b07c86fddd'
223const recordedTraces = unwrapEvents(JSON.parse(readFileSync(new URL('./fixtures/cd/canary-traces.json', import.meta.url), 'utf8')))
224const recordedSpans = unwrapEvents(JSON.parse(readFileSync(new URL('./fixtures/cd/canary-spans.json', import.meta.url), 'utf8')))
226test('recorded query responses exercise every telemetry evaluator', () => {
227 assert.ok(recordedTraces.length > 0)
228 assert.ok(recordedTraces.every(row => {
229 const rec = asRecord(row)
230 return rec !== null && isTraceSummary(rec) && typeof rec.spans === 'number'
231 }))
232 assert.equal(fetchSample(recordedTraces[0]), null)
233 const samples = recordedSpans.flatMap(span => {
234 const sample = fetchSample(span)
235 return sample ? [sample] : []
236 })
237 const upload = samples.filter(sample => sample.version === CANARY && sample.path.endsWith('/git-upload-pack') && sample.status === 200)
238 const receive = samples.filter(sample => sample.version === CANARY && sample.path.endsWith('/git-receive-pack') && sample.status === 200)
239 const refs = samples.filter(sample => sample.path.endsWith('/info/refs') && sample.status === 200)
240 assert.equal(upload.length, 160)
241 assert.equal(percentile(upload.map(sample => sample.durationMs), 0.5), 108)
242 assert.equal(upload.some(sample => sample.durationMs === 523), true)
243 assert.equal(receive.length, 36)
244 assert.equal(percentile(receive.map(sample => sample.durationMs), 0.5), 107)
245 assert.equal(receive.some(sample => sample.durationMs === 691), true)
246 assert.equal(refs.length, 1)
247 assert.equal(refs[0]?.durationMs, 49)
248 const cursor = pageCursor(recordedSpans)
249 const metaIds = recordedSpans.map(span => unwrapOr(dig(span, ['$metadata', 'id']), null))
250 const steppedBack = metaIds.filter(id => id !== metaIds[metaIds.length - 1]).at(-1)
251 assert.equal(cursor, steppedBack)
253 const requests: SmokeRequest[] = []
254 const events: unknown[] = []
255 for (const row of recordedTraces) {
256 const traceId = asRecord(row)?.traceId
257 if (typeof traceId !== 'string') continue
258 const requestId = `req-${traceId.slice(0, 12)}`
259 requests.push({ id: traceId, version: CANARY, step: 'commit', status: 200, expected: [200], ray: null })
260 events.push({
261 $metadata: { id: `log-${traceId}`, requestId, traceId, service: 'ply', level: 'info' },
262 $workers: { scriptVersion: { id: CANARY }, outcome: 'ok' },
263 source: { msg: 'ply.request', id: traceId, status: 200, method: 'POST', path: '/v1/commit', ms: 80 },
264 })
265 events.push({
266 $metadata: { id: `inv-${traceId}`, requestId, traceId, type: 'cf-worker-event', service: 'ply' },
267 $workers: { scriptVersion: { id: CANARY }, eventType: 'fetch', outcome: 'ok', wallTimeMs: 80, cpuTimeMs: 12 },
268 })
269 }
270 const blinded = summarizeTelemetry(CANARY, requests, events, recordedTraces)
271 assert.equal(blinded.spanShape, 'summaries')
272 assert.equal(blinded.fetchMs.length, 0)
273 assert.equal(blinded.tracedIds.size, requests.length)
274 const blindedJudge = judgeSmokeTelemetry({ version: CANARY, requests, summary: blinded, ceilings: SMOKE_LATENCY, maxLossRate: null })
275 assert.equal(blindedJudge.kind, 'harness_fault')
276 if (blindedJudge.kind === 'harness_fault') assert.match(blindedJudge.reason, /no child spans/)
278 const seen = summarizeTelemetry(CANARY, requests, events, recordedTraces, recordedSpans)
279 assert.equal(seen.spanShape, 'events')
280 assert.equal(seen.fetchMs.length, 200)
281 assert.ok((percentile(seen.fetchMs, 0.99) ?? 0) < SMOKE_LATENCY.fetchMs.p99Ms)
282 const judge = judgeSmokeTelemetry({ version: CANARY, requests, summary: seen, ceilings: SMOKE_LATENCY, maxLossRate: null })
283 assert.equal(judge.kind, 'pass')
284 const versionStats = statsFromEvents(events, recordedSpans)
285 assert.equal(versionStats.fetchMs.length, 200)
286})
288test('the revoke probe URL does not carry the token', () => {
289 const token = 'art_v2_super-secret-token-value'
290 const remote = 'https://abc.artifacts.cloudflare.net/git/ply-loadgen/ply-web-cd-test.git'
291 const url = artifactsProbeUrl(remote)
292 assert.equal(url.includes(token), false)
293 assert.equal(url, `${remote}/info/refs?service=git-upload-pack`)
294 assert.equal(revokeConfirmed(403), true)
295 assert.equal(revokeConfirmed(200), false)
296 assert.equal(revokeConfirmed(401), false)
297 assert.equal(revokeConfirmed(0), false)
298})
300/** The 10% flag-service rollback. Named entrypoints were counted as routed traffic. */
301const NEW_VERSION = '3a56fa4f-7b3d-4c1a-9e20-6f0a1b2c3d4e'
302const OLD_VERSION = '11111111-2222-4333-8444-555566667777'
303const NEW_EXTERNAL = 203
304const NEW_FLAGS = 92
305const OLD_EXTERNAL = 1774
306const OLD_GIT = 1
307const FLAGS_CACHE = 'FlagsCache'
308const GIT_CACHE = 'GitCache'
309const INTERNAL_WALL_MS = 99_000
310const LOGGED_MS = 12
311const EXTERNAL_FETCH_MS = 20
313function percentText(fraction: number | null): string {
314 if (fraction === null) return '-'
315 return (fraction * 100).toFixed(2)
316}
318function eventIdentity(n: number) {
319 const hex = n.toString(16).padStart(12, '0')
320 return {
321 requestId: `${hex.slice(0, 8)}-0000-4000-8000-${hex}`,
322 traceId: hex.padStart(32, 'c'),
323 rayId: `${hex.padStart(16, 'd')}-IAD`,
324 }
325}
327function workerFetch(
328 version: string,
329 n: number,
330 entrypoint: string | null,
331 workersOver: Record<string, unknown> = {},
332) {
333 const id = eventIdentity(n)
334 const workers: Record<string, unknown> = {
335 scriptVersion: { id: version },
336 eventType: 'fetch',
337 outcome: 'ok',
338 wallTimeMs: 40,
339 cpuTimeMs: 8,
340 ...workersOver,
341 }
342 if (entrypoint !== null) workers.entrypoint = entrypoint
343 return {
344 $metadata: {
345 id: `inv-${id.requestId}`,
346 requestId: id.requestId,
347 traceId: id.traceId,
348 type: 'cf-worker-event',
349 service: 'ply',
350 },
351 rayId: id.rayId,
352 traceId: id.traceId,
353 $workers: workers,
354 }
355}
357test('named entrypoints are not routed traffic', () => {
358 const bare = workerFetch(NEW_VERSION, 1, null)
359 const namedDefault = workerFetch(NEW_VERSION, 2, DEFAULT_ENTRYPOINT)
360 const flags = workerFetch(NEW_VERSION, 3, FLAGS_CACHE)
361 const classified = fetchInvocation(flags)
362 assert.equal(fetchInvocation(bare)?.kind, 'external')
363 assert.equal(fetchInvocation(namedDefault)?.kind, 'external')
364 assert.equal(classified?.kind, 'internal')
365 if (classified?.kind === 'internal') assert.equal(classified.entrypoint, FLAGS_CACHE)
366 assert.equal(fetchInvocation({ $metadata: { type: 'span' }, $workers: { eventType: 'fetch' } }), null)
368 const events: ReturnType<typeof workerFetch>[] = []
369 for (let n = 0; n < NEW_EXTERNAL; n++) events.push(workerFetch(NEW_VERSION, n, n === 0 ? null : DEFAULT_ENTRYPOINT))
370 const flagsStart = NEW_EXTERNAL
371 for (let n = 0; n < NEW_FLAGS; n++) {
372 const extra = n === 0
373 ? { outcome: 'exception', exceptions: [{ name: 'Error' }], wallTimeMs: INTERNAL_WALL_MS, cpuTimeMs: INTERNAL_WALL_MS }
374 : {}
375 events.push(workerFetch(NEW_VERSION, flagsStart + n, FLAGS_CACHE, extra))
376 }
377 const oldStart = flagsStart + NEW_FLAGS
378 for (let n = 0; n < OLD_EXTERNAL; n++) events.push(workerFetch(OLD_VERSION, oldStart + n, DEFAULT_ENTRYPOINT))
379 const gitIndex = oldStart + OLD_EXTERNAL
380 events.push(workerFetch(OLD_VERSION, gitIndex, GIT_CACHE))
382 const requestIds = new Set(events.map(event => event.$metadata.requestId))
383 const rays = new Set(events.map(event => event.rayId))
384 const traces = new Set(events.map(event => event.traceId))
385 const inflated = events.length
386 assert.equal(inflated, NEW_EXTERNAL + NEW_FLAGS + OLD_EXTERNAL + OLD_GIT)
387 assert.equal(requestIds.size, inflated)
388 assert.equal(rays.size, inflated)
389 assert.equal(traces.size, inflated)
391 const flagsId = eventIdentity(flagsStart)
392 const logTrace = 'f'.repeat(32)
393 const logRay = `${'e'.repeat(16)}-IAD`
394 events.push({
395 $metadata: { id: 'log-flags', requestId: flagsId.requestId, traceId: logTrace, type: 'log', service: 'ply', level: 'info' },
396 rayId: logRay,
397 traceId: logTrace,
398 $workers: { scriptVersion: { id: NEW_VERSION }, outcome: 'ok' },
399 source: { msg: 'ply.request', id: 'flags-log', status: 200, method: 'GET', path: '/v1/discover', ms: LOGGED_MS },
400 } as ReturnType<typeof workerFetch>)
402 const externalRay = eventIdentity(0).rayId
403 const internalRay = flagsId.rayId
404 const spans = [
405 { rayId: externalRay, traceId: eventIdentity(0).traceId, name: 'fetch', durationMs: EXTERNAL_FETCH_MS, $metadata: { type: 'span', requestId: null, traceId: eventIdentity(0).traceId } },
406 { rayId: internalRay, traceId: flagsId.traceId, name: 'fetch', durationMs: INTERNAL_WALL_MS, $metadata: { type: 'span', requestId: null, traceId: flagsId.traceId } },
407 ]
408 const onNew = events.filter(event => (event.$workers.scriptVersion as { id: string }).id === NEW_VERSION)
409 const onOld = events.filter(event => (event.$workers.scriptVersion as { id: string }).id === OLD_VERSION)
410 const next = statsFromEvents(onNew, spans)
411 const previous = statsFromEvents(onOld, spans)
412 const aggregate = statsFromEvents(events, spans)
414 assert.equal(next.invocations, NEW_EXTERNAL)
415 assert.equal(previous.invocations, OLD_EXTERNAL)
416 assert.equal(aggregate.invocations, NEW_EXTERNAL + OLD_EXTERNAL)
417 assert.equal(next.requests, NEW_EXTERNAL)
418 assert.equal(next.errors, 0)
419 assert.equal(next.exceptions, 0)
420 assert.equal(next.wallMs.includes(INTERNAL_WALL_MS), false)
421 assert.equal(next.cpuMs.includes(INTERNAL_WALL_MS), false)
422 assert.deepEqual(next.fetchMs, [EXTERNAL_FETCH_MS])
423 assert.deepEqual(next.internal, [{ entrypoint: FLAGS_CACHE, count: NEW_FLAGS }])
424 assert.deepEqual(previous.internal, [{ entrypoint: GIT_CACHE, count: OLD_GIT }])
425 assert.equal(next.v1.length, 1)
426 assert.equal(next.v1[0]?.exception, false)
427 assert.equal(next.v1[0]?.wallMs, LOGGED_MS)
429 const summary = summarizeTelemetry(NEW_VERSION, [], onNew, [])
430 assert.equal(summary.invocations, NEW_EXTERNAL)
431 assert.equal(summary.health.exceptions, 0)
432 assert.equal(summary.health.nonOk, 0)
434 const policy = loadPolicy(DEFAULT_POLICY_PATH).policy
435 const targetPercent = policy.steps[0]?.percent
436 if (targetPercent === undefined) throw new Error('policy has no step')
437 const counted = routedShare(next.invocations, aggregate.invocations, targetPercent, policy.split.z)
438 const asFetched = routedShare(NEW_EXTERNAL + NEW_FLAGS, inflated, targetPercent, policy.split.z)
439 assert.equal(percentText(asFetched.share), '14.25')
440 assert.equal(asFetched.kind, 'mismatch')
441 assert.equal(`${percentText(asFetched.low)}..${percentText(asFetched.high)}`, '8.02..11.98')
442 assert.equal(percentText(counted.share), '10.27')
443 assert.equal(counted.kind, 'match')
444 assert.equal(counted.newRequests, NEW_EXTERNAL)
445 assert.equal(counted.n, NEW_EXTERNAL + OLD_EXTERNAL)
446 assert.ok(counted.share !== null && asFetched.low !== null && asFetched.high !== null)
447 if (counted.share !== null && asFetched.low !== null && asFetched.high !== null) {
448 assert.ok(counted.share >= asFetched.low && counted.share <= asFetched.high)
449 }
450 const decision = judgeStep({ next, previous, aggregate, targetPercent, policy })
451 assert.equal(decision.outcome, 'advance')
452 const line = progressLine({
453 step: 1,
454 steps: policy.steps.length,
455 percent: targetPercent,
456 next,
457 previous,
458 carriedPrevious: null,
459 share: counted,
460 decision,
461 latency: policy.gates.latency.mode,
462 probe: { kind: 'none' },
463 sample: ALL_INVOCATIONS,
464 })
465 assert.match(line, new RegExp(`samples=${NEW_EXTERNAL}/${OLD_EXTERNAL}`))
466 assert.match(line, /routed_share=10.27/)
467 assert.match(line, /tolerance=7.98..12.02/)
468 assert.match(line, new RegExp(`agg=${NEW_EXTERNAL + OLD_EXTERNAL}`))
469 assert.match(line, new RegExp(`internal_next=${FLAGS_CACHE}:${NEW_FLAGS}`))
470 assert.match(line, new RegExp(`internal_previous=${GIT_CACHE}:${OLD_GIT}`))
471 assert.match(line, /decision=advance/)
472 assert.equal(line.includes('reason='), false)
473})
475/** N smoke requests; the first `lost` have no log, invocation or trace. Each traced one has its own ray. */
476function lossFixture(total: number, lost: number, status = 200) {
477 const requests: SmokeRequest[] = []
478 const events: unknown[] = []
479 const spans: unknown[] = []
480 for (let i = 0; i < total; i++) {
481 const hex = i.toString(16).padStart(12, '0')
482 const id = `44444444-4444-4444-8444-${hex}`
483 const requestId = `dddddddd-dddd-4ddd-8ddd-${hex}`
484 const ray = `${hex}abcd-IAD`
485 const trace = `${hex}${'e'.repeat(20)}`
486 requests.push({ ...smoke(id, ray, status), step: `step-${i}` })
487 if (i < lost) continue
488 events.push(logEvent(id, requestId, trace, status), invocation(id, requestId, trace))
489 spans.push(spanEvent('fetch', 20, ray, trace))
490 }
491 return { requests, summary: summarizeTelemetry(VERSION, requests, events, spans) }
492}
494const LOSS_TOTAL = 79
495const LOSS_SEEN = 6
497test('telemetry loss below maxLossRate is informational, with count, rate and ids', () => {
498 const { requests, summary } = lossFixture(LOSS_TOTAL, LOSS_SEEN)
499 const loss = telemetryLoss(requests, summary, DEFAULT_MAX_LOSS_RATE)
500 assert.equal(loss.count, LOSS_SEEN)
501 assert.equal(loss.total, LOSS_TOTAL)
502 assert.equal(loss.rate, LOSS_SEEN / LOSS_TOTAL)
503 assert.equal(loss.exceeded, false)
504 assert.deepEqual(loss.ids.map(lost => lost.step), requests.slice(0, LOSS_SEEN).map(request => request.step))
505 assert.ok(loss.ids.every(lost => lost.log && lost.trace))
506 const judge = judgeSmokeTelemetry({ version: VERSION, requests, summary, ceilings: SMOKE_LATENCY, maxLossRate: DEFAULT_MAX_LOSS_RATE })
507 assert.equal(judge.kind, 'pass')
508 assert.match(judge.line, /telemetry loss 6\/79 .* informational/)
509})
511test('telemetry loss above maxLossRate is a harness fault', () => {
512 const lost = Math.ceil(LOSS_TOTAL * DEFAULT_MAX_LOSS_RATE) + 1
513 const { requests, summary } = lossFixture(LOSS_TOTAL, lost)
514 const judge = judgeSmokeTelemetry({ version: VERSION, requests, summary, ceilings: SMOKE_LATENCY, maxLossRate: DEFAULT_MAX_LOSS_RATE })
515 assert.equal(judge.kind, 'harness_fault')
516 if (judge.kind === 'harness_fault') assert.match(judge.reason, /telemetry-loss: telemetry loss/)
517})
519test('a tight maxLossRate fails the same loss the default tolerates', () => {
520 const { requests, summary } = lossFixture(LOSS_TOTAL, LOSS_SEEN)
521 const tight = (LOSS_SEEN - 1) / LOSS_TOTAL
522 const judge = judgeSmokeTelemetry({ version: VERSION, requests, summary, ceilings: SMOKE_LATENCY, maxLossRate: tight })
523 assert.equal(judge.kind, 'harness_fault')
524})
526test('loss never hides a correctness fault: a 5xx with loss is still a worker fault', () => {
527 const { requests, summary } = lossFixture(LOSS_TOTAL, LOSS_SEEN, 500)
528 const judge = judgeSmokeTelemetry({ version: VERSION, requests, summary, ceilings: SMOKE_LATENCY, maxLossRate: DEFAULT_MAX_LOSS_RATE })
529 assert.equal(judge.kind, 'worker')
530})
Diagnostics4 git calls · 186 ms
CallStatusSizeTime0186 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2006.2 KiB181 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=181 git.response.bytes=6321 git.response.cache=MISS git.response.duration_ms=181 git.fetch.wants=60b7d2237121a0ed67c097269551e4810d0e8ef7 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=6321 git.pkt.payload.max=6300 git.sideband.band1.bytes=6299 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2e41ebad92947
- Request
- 0c78f16b…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format