main
ply4 git calls29 ms49 KiB3 cachedprotocol v2
797 lines30 KiBRaw
1import { mkdtemp, rm } from 'node:fs/promises'
2import { tmpdir } from 'node:os'
3import { join } from 'node:path'
4import { REPO_PREFIX, REVOKE_ATTEMPTS, REVOKE_PROBE_MS, REVOKE_WAIT_MS, SMOKE_HTTP_MS, SMOKE_REISSUE_ATTEMPTS, SMOKE_REISSUE_WAIT_MS, SMOKE_RESTARTS, TOKEN_TTL_SECONDS } from './baseline.ts'
5import { clientHealth, type StallSource } from './client-health.ts'
6import { confirmRoute, failureFault, logRoute, ROUTE_TUNABLES, SYSTEM_ROUTE_IO, type RouteIo, type RouteTunables } from './route-confirm.ts'
7import { cfOk, runCf } from './cf.ts'
8import {
9 appendCleanupFile,
10 confirmDelete,
11 recordCleanup,
12 reposFromList,
13 sweepLeaks,
14 type CleanupBound,
15 type CleanupResult,
16 type DeleteCommand,
17 type SweepReport,
18} from './cleanup.ts'
19import { artifactsProbeUrl, classifyCall, openingRevokeResult, revokeConfirmed, type FailureClass } from './classify.ts'
20import { csvDocument, type CsvRow } from './csv.ts'
21import { asRecord, extractJson } from './json.ts'
22import { judgeCall, type Saved } from './judge.ts'
23import { expectedStatuses, lifecycle, type HttpStep } from './lifecycle.ts'
24import { activeLog } from './log.ts'
25import { assertNever } from '../../src/never.ts'
26import { none, some, type Option } from '../../src/option.ts'
27import { errorMessage, run } from './proc.ts'
28import { createdTokenSecret, harvestSecrets, redact } from './redact.ts'
29import { judgeFsck, judgeRefs, parseShowRef, type ApiRefs } from './refs.ts'
30import { fail, pass, type StageResult } from './result.ts'
31import type { Secret } from './secret.ts'
32import { writeJson, writeText } from './run-dir.ts'
33import { CORRECTNESS } from './slo.ts'
34import type { SmokeRequest } from './telemetry.ts'
35import type { CdConfig } from './config.ts'
37export type SmokeReport = {
38 readonly version: string | null
39 readonly repo: string
40 readonly deleted: boolean
41 /** Scratch repos whose delete could not be confirmed. Informational: it does not fail a pass. */
42 readonly cleanup_leak: readonly string[]
43 readonly requests: readonly SmokeRequest[]
44 readonly refs: ApiRefs | null
45 readonly detail: string
46 readonly passed: number
47 readonly failed: number
48 /** Rows answered by another version or lost while the client stalled: re-issued or restarted, never counted. */
49 readonly harnessEvents: number
50 /** Lifecycle restarts on a new scratch repo after a write met a harness event. */
51 readonly restarts: number
52 /** Scratch repos of discarded attempts (deleted). */
53 readonly discarded: readonly string[]
56export type SweepRun =
57 | { readonly kind: 'swept'; readonly deleted: readonly string[]; readonly leaks: readonly string[] }
58 | { readonly kind: 'unlistable'; readonly detail: string }
60const REMOTE = /^https:\/\/[a-z0-9]+\.artifacts\.cloudflare\.net\/git\//
62function header(response: Response, name: string): string | null {
63 const value = response.headers.get(name)
64 return value === null || value === '' ? null : value
67/** The first Artifacts git remote anywhere in a `cf` JSON answer; none when it carries none. */
68export function findRemote(value: unknown): Option<string> {
69 if (typeof value === 'string') return REMOTE.test(value) ? some(value) : none()
70 const items: readonly unknown[] = Array.isArray(value) ? value : objectValues(value)
71 for (const item of items) {
72 const found = findRemote(item)
73 if (found.tag === 'some') return found
74 }
75 return none()
78function objectValues(value: unknown): readonly unknown[] {
79 const rec = asRecord(value)
80 return rec ? Object.values(rec) : []
83function tokenIds(value: unknown): string[] {
84 const ids: string[] = []
85 const visit = (node: unknown) => {
86 if (Array.isArray(node)) {
87 for (const item of node) visit(item)
88 return
89 }
90 const rec = asRecord(node)
91 if (!rec) return
92 if (typeof rec.id === 'string' && rec.id !== '') ids.push(rec.id)
93 for (const item of Object.values(rec)) visit(item)
94 }
95 visit(value)
96 return ids
99function remember(value: unknown, secrets: string[]): ReturnType<typeof harvestSecrets> {
100 const found = harvestSecrets(value)
101 for (const secret of found) {
102 secrets.push(secret.reveal())
103 activeLog()?.concealSecret(secret)
104 }
105 return found
108async function cfJson(args: readonly string[], secrets: readonly string[]): Promise<unknown> {
109 const stdout = await cfOk(args, secrets)
110 return extractJson(stdout)
113/** After revoke, the minted bearer must be refused. The token is sent and never recorded. */
114async function proveRevoked(remote: string, secret: Secret): Promise<boolean> {
115 const url = artifactsProbeUrl(remote)
116 const started = Date.now()
117 let status = 0
118 let cfRay: string | null = null
119 try {
120 const response = await fetch(url, {
121 method: 'GET',
122 headers: { authorization: `Bearer ${secret.reveal()}` },
123 redirect: 'manual',
124 signal: AbortSignal.timeout(REVOKE_PROBE_MS),
125 })
126 status = response.status
127 cfRay = header(response, 'cf-ray')
128 try {
129 await response.body?.cancel()
130 } catch {
131 // The status is what the check records. The body is not.
132 }
133 } catch {
134 status = 0
135 }
136 await activeLog()?.request({
137 method: 'GET',
138 url,
139 status,
140 version: null,
141 latencyMs: Date.now() - started,
142 cfRay,
143 requestId: null,
144 artifactsRay: null,
145 })
146 return revokeConfirmed(status)
149async function revoke(config: CdConfig, id: string, secrets: readonly string[]): Promise<boolean> {
150 for (let attempt = 0; attempt < REVOKE_ATTEMPTS; attempt++) {
151 const result = await runCf(['artifacts', 'namespaces', 'tokens', 'revoke', id, '--namespace', config.namespace, '--force'], secrets)
152 if (result.code === 0) return true
153 await new Promise(resolve => setTimeout(resolve, REVOKE_WAIT_MS))
154 }
155 return false
158async function listTokenIds(config: CdConfig, name: string, secrets: readonly string[]): Promise<string[]> {
159 const listed = await runCf([
160 'artifacts', 'namespaces', 'repos', 'tokens', 'list',
161 '--namespace', config.namespace, '--name', name, '--state', 'active',
162 ], secrets)
163 if (listed.code !== 0) return []
164 try {
165 return tokenIds(extractJson(listed.stdout))
166 } catch {
167 return []
168 }
171function cleanupBound(config: CdConfig): CleanupBound {
172 return { attempts: config.policy.cleanup.attempts, backoffMs: config.policy.cleanup.backoffMs }
175function wait(ms: number): Promise<void> {
176 return new Promise(resolve => {
177 setTimeout(resolve, ms)
178 })
181async function deleteOnce(name: string, config: CdConfig, secrets: readonly string[]): Promise<DeleteCommand> {
182 const started = Date.now()
183 const removed = await runCf(['artifacts', 'namespaces', 'repos', 'delete', name, '--namespace', config.namespace, '--force'], secrets)
184 return { code: removed.code, stdout: removed.stdout, stderr: removed.stderr, latencyMs: Date.now() - started }
187async function noteCleanup(result: CleanupResult, bound: CleanupBound): Promise<void> {
188 const log = activeLog()
189 if (!log) return
190 await recordCleanup(log, result, bound)
193async function sweep(config: CdConfig, secrets: readonly string[]): Promise<SweepReport | null> {
194 const listed = await runCf(['artifacts', 'namespaces', 'repos', 'list', '--namespace', config.namespace, '--search', REPO_PREFIX], secrets)
195 if (listed.code !== 0) return null
196 let parsed: unknown
197 try {
198 parsed = extractJson(listed.stdout)
199 } catch {
200 return null
201 }
202 const bound = cleanupBound(config)
203 return sweepLeaks({
204 listed: reposFromList(parsed),
205 nowMs: Date.now(),
206 staleMs: config.policy.cleanup.staleMinutes * 60_000,
207 prefix: REPO_PREFIX,
208 bound,
209 once: name => deleteOnce(name, config, secrets),
210 sleep: wait,
211 note: result => noteCleanup(result, bound),
212 })
215/** Delete leaked `ply-web-cd-*` repos older than `cleanup.staleMinutes`. A leak stays informational. */
216export async function stageSweep(config: CdConfig, runDir: string): Promise<SweepRun> {
217 const listed = await runCf(['artifacts', 'namespaces', 'repos', 'list', '--namespace', config.namespace, '--search', REPO_PREFIX], [])
218 if (listed.code !== 0) {
219 const detail = listed.stderr.replace(/\s+/g, ' ').trim().slice(0, 300)
220 return { kind: 'unlistable', detail: detail.length === 0 ? `cf list exited ${listed.code}` : detail }
221 }
222 let parsed: unknown
223 try {
224 parsed = extractJson(listed.stdout)
225 } catch (error) {
226 return { kind: 'unlistable', detail: errorMessage(error) }
227 }
228 const bound = cleanupBound(config)
229 const report = await sweepLeaks({
230 listed: reposFromList(parsed),
231 nowMs: Date.now(),
232 staleMs: config.policy.cleanup.staleMinutes * 60_000,
233 prefix: REPO_PREFIX,
234 bound,
235 once: name => deleteOnce(name, config, []),
236 sleep: wait,
237 note: result => noteCleanup(result, bound),
238 })
239 await appendCleanupFile(runDir, report.leaks)
240 await writeJson(runDir, 'verdict.json', {
241 kind: 'pass',
242 failures: [],
243 cleanup_leak: report.leaks,
244 deleted: report.deleted,
245 })
246 return { kind: 'swept', deleted: report.deleted, leaks: report.leaks }
249function stepPath(step: HttpStep): string {
250 const params = new URLSearchParams(step.query ?? {})
251 const query = params.toString()
252 return query ? `${step.path}?${query}` : step.path
255export type CallResult = {
256 readonly started: number
257 readonly ended: number
258 /** `x-ply-version` of the response, null when there was none (or no response). */
259 readonly served: string | null
260 readonly row: CsvRow
261 readonly judgementDetail: string
262 readonly ok: boolean
263 readonly save: Saved
264 readonly request: SmokeRequest | null
265 readonly refs: ApiRefs | null
266 readonly fault: FailureClass | null
269/**
270 * A row that is not evidence about the version under test. `wrong_version`: another version answered (the override or
271 * the new split was not yet on that edge machine). `client_stall`: no response while this process was stalled
272 * (client-health.ts). Either is re-issued or restarts the attempt; neither is a pass or a product failure.
273 */
274export type HarnessEvent =
275 | { readonly kind: 'wrong_version'; readonly served: string; readonly detail: string }
276 | { readonly kind: 'client_stall'; readonly stalledMs: number; readonly detail: string }
278export function harnessEvent(
279 call: { readonly started: number; readonly ended: number; readonly served: string | null; readonly status: number },
280 version: string | null,
281 health: StallSource,
282): HarnessEvent | null {
283 if (call.status !== 0 && version !== null && call.served !== null && call.served !== version) {
284 return { kind: 'wrong_version', served: call.served, detail: `routing: answered by ${call.served}, wanted ${version}` }
285 }
286 if (call.status === 0) {
287 const stalls = health.during(call.started, call.ended)
288 if (stalls.length > 0) {
289 const stalledMs = health.stalledMs(call.started, call.ended)
290 return { kind: 'client_stall', stalledMs, detail: `client stall: no response while the harness was stalled ${stalledMs} ms` }
291 }
292 }
293 return null
296/** Re-issue a GET; restart the attempt (fresh scratch repo) for anything else, since a write may have landed. */
297export function reissuable(step: { readonly method: string }): boolean {
298 return step.method === 'GET'
301async function oneCall(
302 config: CdConfig,
303 step: HttpStep,
304 saved: Saved,
305 remote: string,
306 token: string,
307 gitToken: string,
308 version: string | null,
309 pin: boolean,
310): Promise<CallResult> {
311 const started = Date.now()
312 const params = new URLSearchParams({ remote, ...(step.query ?? {}) })
313 const url = `${config.baseUrl}${step.path}?${params.toString()}`
314 const headers: Record<string, string> = {}
315 if (step.ply !== false) headers.authorization = `Bearer ${token}`
316 if (step.git !== false) headers['git-token'] = gitToken
317 if (pin && version) headers['Cloudflare-Workers-Version-Overrides'] = `${config.worker}="${version}"`
318 const rawBody = step.body?.(saved)
319 if (rawBody) headers['content-type'] = 'application/json'
320 let response: Response
321 try {
322 response = await fetch(url, {
323 method: step.method,
324 headers,
325 body: rawBody ? JSON.stringify({ ...rawBody, remote }) : undefined,
326 signal: AbortSignal.timeout(SMOKE_HTTP_MS),
327 })
328 } catch (error) {
329 const detail = redact(errorMessage(error), [token, gitToken])
330 await activeLog()?.request({
331 method: step.method, url, status: 0, version: null, latencyMs: Date.now() - started,
332 cfRay: null, requestId: null, artifactsRay: null,
333 })
334 return {
335 started,
336 ended: Date.now(),
337 served: null,
338 row: {
339 step: step.name, method: step.method, path: stepPath(step), status: 0, ms: Date.now() - started,
340 pass: false, cfRay: '', requestId: '', artifactsRay: '', version: '', build: '', serverTiming: '', detail,
341 },
342 judgementDetail: detail,
343 ok: false,
344 save: {},
345 request: null,
346 refs: null,
347 fault: classifyCall({ status: 0, body: detail, timedOut: true, mintedToken: step.git !== false }),
348 }
349 }
350 const bodyText = await response.text()
351 const requestId = header(response, 'x-ply-request-id')
352 const build = header(response, 'x-ply-build')
353 const seen = header(response, 'x-ply-version')
354 const ray = header(response, 'x-artifacts-ray')
355 const judgement = judgeCall({
356 status: response.status,
357 bodyText,
358 headers: {
359 requestId,
360 build,
361 version: seen,
362 ray,
363 cacheControl: header(response, 'cache-control'),
364 contentType: header(response, 'content-type'),
365 },
366 ray: step.ray,
367 rayIncludes: step.rayIncludes,
368 version: pin || version ? version : null,
369 expect: step.expect,
370 save: step.save,
371 secrets: [token, gitToken],
372 }, saved)
373 let refs: ApiRefs | null = null
374 if (step.name === 'refs-final' && judgement.kind === 'ok') {
375 try {
376 const parsed = JSON.parse(bodyText) as unknown
377 const rec = asRecord(parsed)
378 const map = asRecord(rec?.refs)
379 if (rec && map) {
380 const peeled = asRecord(rec.peeled) ?? {}
381 refs = {
382 head: typeof rec.head === 'string' ? rec.head : null,
383 refs: Object.fromEntries(Object.entries(map).filter((entry): entry is [string, string] => typeof entry[1] === 'string')),
384 peeled: Object.fromEntries(Object.entries(peeled).filter((entry): entry is [string, string] => typeof entry[1] === 'string')),
385 }
386 }
387 } catch {
388 refs = null
389 }
390 }
391 const detail = judgement.kind === 'ok' ? 'ok' : judgement.detail
392 await activeLog()?.request({
393 method: step.method,
394 url,
395 status: response.status,
396 version: seen,
397 latencyMs: Date.now() - started,
398 cfRay: header(response, 'cf-ray'),
399 requestId,
400 artifactsRay: ray,
401 })
402 return {
403 started,
404 ended: Date.now(),
405 served: seen,
406 row: {
407 step: step.name,
408 method: step.method,
409 path: stepPath(step),
410 status: response.status,
411 ms: Date.now() - started,
412 pass: judgement.kind === 'ok',
413 cfRay: header(response, 'cf-ray') ?? '',
414 requestId: requestId ?? '',
415 artifactsRay: ray ?? '',
416 version: seen ?? '',
417 build: build ?? '',
418 serverTiming: header(response, 'server-timing') ?? '',
419 detail,
420 },
421 judgementDetail: detail,
422 ok: judgement.kind === 'ok',
423 save: judgement.kind === 'ok' ? judgement.save : {},
424 request: requestId ? { id: requestId, version: seen, step: step.name, status: response.status, expected: expectedStatuses(step), ray: header(response, 'cf-ray') } : null,
425 refs,
426 fault: judgement.kind === 'ok' ? null : classifyCall({
427 status: response.status,
428 body: bodyText,
429 timedOut: response.status === 0,
430 mintedToken: step.git !== false,
431 }),
432 }
435async function judgeMirror(remote: string, gitToken: string, api: ApiRefs): Promise<string | null> {
436 const dest = await mkdtemp(join(tmpdir(), 'ply-cd-'))
437 const secrets = [gitToken]
438 const env = {
439 ...process.env,
440 GIT_TERMINAL_PROMPT: '0',
441 GIT_CONFIG_COUNT: '1',
442 GIT_CONFIG_KEY_0: 'http.extraHeader',
443 GIT_CONFIG_VALUE_0: `Authorization: Bearer ${gitToken}`,
444 }
445 try {
446 const cloned = await run('git', ['clone', '--mirror', remote, dest], { env, secrets })
447 if (cloned.code !== 0) return `git clone failed: ${cloned.stderr.slice(0, 300)}`
448 const fsck = await run('git', ['fsck', '--strict', '--full'], { cwd: dest, env, secrets })
449 const fsckJudgement = judgeFsck(fsck.code, fsck.stdout, fsck.stderr)
450 if (!fsckJudgement.ok) return fsckJudgement.detail
451 const shown = await run('git', ['show-ref'], { cwd: dest, env, secrets })
452 if (shown.code !== 0) return `git show-ref failed: ${shown.stderr.slice(0, 300)}`
453 const showRef = parseShowRef(shown.stdout)
454 const head = await run('git', ['symbolic-ref', 'HEAD'], { cwd: dest, env, secrets })
455 if (head.code !== 0) return `git symbolic-ref failed: ${head.stderr.slice(0, 300)}`
456 const peeled = new Map<string, string>()
457 for (const name of showRef.keys()) {
458 const parsed = await run('git', ['rev-parse', `${name}^{}`], { cwd: dest, env, secrets })
459 if (parsed.code !== 0) return `git rev-parse ${name} failed: ${parsed.stderr.slice(0, 200)}`
460 peeled.set(name, parsed.stdout.trim())
461 }
462 const refs = judgeRefs(api, showRef, peeled, head.stdout.trim())
463 return refs.ok ? null : refs.detail
464 } finally {
465 await rm(dest, { recursive: true, force: true })
466 }
469type HarnessFaultKind = 'harness_fault' | 'harness_unhealthy'
471type AttemptResult = {
472 readonly name: string
473 readonly deleted: boolean
474 readonly refs: ApiRefs | null
475 readonly detail: string
476 readonly passed: boolean
477 readonly harness: boolean
478 readonly fault: HarnessFaultKind | null
479 /** Set when a write met a harness event: discard this scratch repo and run the lifecycle again on a new one. */
480 readonly restart: string | null
483/** A harness event's row: kept in the CSV for the record, never counted as a pass or a failure. */
484function eventRow(row: CsvRow, event: HarnessEvent, next: string): CsvRow {
485 return { ...row, pass: false, detail: `${event.detail}; ${next}` }
488async function noteEvent(step: HttpStep, event: HarnessEvent, call: CallResult, next: string): Promise<void> {
489 const log = activeLog()
490 if (!log) return
491 await log.observation({
492 measurement: {
493 source: 'smoke-harness-event',
494 step: step.name,
495 method: step.method,
496 kind: event.kind,
497 served: event.kind === 'wrong_version' ? event.served : null,
498 stalledMs: event.kind === 'client_stall' ? event.stalledMs : null,
499 status: call.row.status,
500 cfRay: call.row.cfRay,
501 next,
502 },
503 query: { source: 'smoke' },
504 from: call.started,
505 to: call.ended,
506 raw: JSON.stringify({ step: step.name, detail: event.detail, next }),
507 })
510function exhausted(event: HarnessEvent, step: HttpStep, tries: number): { readonly detail: string; readonly fault: HarnessFaultKind } {
511 if (event.kind === 'client_stall') {
512 return { detail: `harness unhealthy, prod untouched: ${step.name}: ${event.detail} on ${tries} tries`, fault: 'harness_unhealthy' }
513 }
514 return { detail: `${step.name}: ${event.detail} on ${tries} tries`, fault: 'harness_fault' }
517async function smokeAttempt(
518 config: CdConfig,
519 opts: { readonly version: string | null; readonly pin: boolean; readonly token: string },
520 secrets: string[],
521 rows: CsvRow[],
522 harnessRows: Set<CsvRow>,
523 requests: SmokeRequest[],
524): Promise<AttemptResult> {
525 const name = `${REPO_PREFIX}${new Date().toISOString().replace(/\D/g, '').slice(0, 14)}-${Math.random().toString(36).slice(2, 8)}`
526 let gitToken = ''
527 let remote = ''
528 const minted: string[] = []
529 let mintedToken: { readonly id: string; readonly secret: Secret } | null = null
530 let deleted = false
531 let saved: Saved = {}
532 let detail = 'ok'
533 let refs: ApiRefs | null = null
534 let passed = false
535 let harness = false
536 let fault: HarnessFaultKind | null = null
537 let restart: string | null = null
538 try {
539 const created = await cfJson([
540 'artifacts', 'namespaces', 'repos', 'create', config.namespace,
541 '--name', name, '--default-branch', 'main', '--description', 'ply-web operator smoke (deleted after a pass)',
542 ], secrets)
543 const found = remember(created, secrets)
544 const auto = createdTokenSecret(created, found)
545 const createdRemote = findRemote(created)
546 if (createdRemote.tag === 'none') throw new Error('cf did not return an Artifacts remote')
547 remote = createdRemote.value
548 const opening = await listTokenIds(config, name, secrets)
549 let revoked = opening.length > 0
550 for (const id of opening) {
551 if (!(await revoke(config, id, secrets))) revoked = false
552 }
553 let confirmed = false
554 if (auto && revoked) confirmed = await proveRevoked(remote, auto)
555 const openingResult = openingRevokeResult({
556 hadToken: auto !== null,
557 listed: opening.length,
558 revoked,
559 confirmed,
560 })
561 if (!openingResult.ok) {
562 detail = `${openingResult.reason} for ${name}`
563 harness = true
564 } else {
565 const mintedJson = await cfJson([
566 'artifacts', 'namespaces', 'tokens', 'create', config.namespace,
567 '--repo', name, '--scope', 'write', '--ttl', String(TOKEN_TTL_SECONDS),
568 ], secrets)
569 const mintedSecrets = remember(mintedJson, secrets)
570 const rec = asRecord(mintedJson)
571 const plain = rec !== null && typeof rec.plaintext === 'string' ? rec.plaintext : ''
572 const writeToken = mintedSecrets.find(secret => secret.reveal() === plain)
573 if (!writeToken) throw new Error('cf returned no write token')
574 gitToken = writeToken.reveal()
575 if (rec !== null && typeof rec.id === 'string') {
576 minted.push(rec.id)
577 mintedToken = { id: rec.id, secret: writeToken }
578 }
579 steps: for (const step of lifecycle()) {
580 let result = await oneCall(config, step, saved, remote, opts.token, gitToken, opts.version, opts.pin)
581 let tries = 1
582 for (;;) {
583 const event = harnessEvent({ started: result.started, ended: result.ended, served: result.served, status: result.row.status }, opts.version, clientHealth())
584 if (event === null) break
585 if (!reissuable(step)) {
586 const next = 'restart the lifecycle on a new scratch repo'
587 const row = eventRow(result.row, event, next)
588 rows.push(row)
589 harnessRows.add(row)
590 await noteEvent(step, event, result, next)
591 restart = `${step.name}: ${event.detail}`
592 if (event.kind === 'client_stall') fault = 'harness_unhealthy'
593 break steps
594 }
595 if (tries > SMOKE_REISSUE_ATTEMPTS) {
596 const next = 'no re-issue left'
597 const row = eventRow(result.row, event, next)
598 rows.push(row)
599 harnessRows.add(row)
600 await noteEvent(step, event, result, next)
601 const out = exhausted(event, step, tries)
602 detail = out.detail
603 fault = out.fault
604 harness = true
605 break steps
606 }
607 const next = `re-issued (${tries}/${SMOKE_REISSUE_ATTEMPTS})`
608 const row = eventRow(result.row, event, next)
609 rows.push(row)
610 harnessRows.add(row)
611 await noteEvent(step, event, result, next)
612 await wait(SMOKE_REISSUE_WAIT_MS)
613 tries += 1
614 result = await oneCall(config, step, saved, remote, opts.token, gitToken, opts.version, opts.pin)
615 }
616 rows.push(result.row)
617 if (result.request) requests.push(result.request)
618 if (!result.ok) {
619 detail = `${step.name}: ${result.judgementDetail}`
620 harness = result.fault === 'harness_fault'
621 break
622 }
623 saved = { ...saved, ...result.save }
624 if (result.refs) refs = result.refs
625 }
626 if (detail === 'ok' && restart === null) {
627 if (!refs) detail = 'refs-final did not return a ref map'
628 else {
629 const gitDetail = await judgeMirror(remote, gitToken, refs)
630 if (gitDetail) detail = gitDetail
631 }
632 }
633 passed = detail === 'ok' && restart === null
634 }
635 } catch (error) {
636 detail = redact(errorMessage(error), secrets)
637 harness = true
638 } finally {
639 const pending = [...new Set([...minted, ...await listTokenIds(config, name, secrets)])]
640 const stuck = []
641 for (const id of pending) {
642 if (!(await revoke(config, id, secrets))) {
643 stuck.push(id)
644 continue
645 }
646 if (mintedToken && mintedToken.id === id && REMOTE.test(remote) && !(await proveRevoked(remote, mintedToken.secret))) {
647 stuck.push(id)
648 }
649 }
650 if ((passed || restart !== null) && stuck.length > 0) {
651 detail = `smoke ${passed ? 'passed' : 'restart'} but a token for ${name} was not revoked`
652 passed = false
653 restart = null
654 harness = true
655 fault = 'harness_fault'
656 }
657 }
658 return { name, deleted, refs, detail, passed, harness, fault, restart }
661/** The route the smoke needs, confirmed again before a restart: the override for a pinned smoke, the split otherwise. */
662export async function reconfirmRoute(
663 config: Pick<CdConfig, 'baseUrl' | 'worker'>,
664 version: string | null,
665 pin: boolean,
666 io: RouteIo = SYSTEM_ROUTE_IO,
667 tunables: RouteTunables = ROUTE_TUNABLES,
668): Promise<{ readonly ok: true } | { readonly ok: false; readonly reason: string; readonly fault: 'harness_fault' | 'worker' }> {
669 if (version === null) return { ok: true }
670 const since = io.now()
671 const verdict = await confirmRoute({
672 url: `${config.baseUrl}/`,
673 version,
674 headers: pin ? { 'Cloudflare-Workers-Version-Overrides': `${config.worker}="${version}"` } : {},
675 since,
676 }, io, tunables)
677 await logRoute({ purpose: pin ? 'pin' : 'live', version, since, started: since }, verdict)
678 if (verdict.kind === 'confirmed') return { ok: true }
679 return { ok: false, reason: verdict.reason, fault: failureFault(verdict) }
682export async function stageSmoke(
683 config: CdConfig,
684 opts: { readonly version: string | null; readonly pin: boolean; readonly token: string; readonly runDir: string },
685): Promise<StageResult<SmokeReport>> {
686 const secrets = [opts.token]
687 const smokeStarted = Date.now()
688 activeLog()?.conceal(opts.token)
689 const leaks: string[] = []
690 const swept = await sweep(config, secrets)
691 if (swept) {
692 for (const repoName of swept.leaks) leaks.push(repoName)
693 }
694 const rows: CsvRow[] = []
695 const harnessRows = new Set<CsvRow>()
696 const requests: SmokeRequest[] = []
697 const discarded: string[] = []
698 let attempt: AttemptResult
699 let restarts = 0
700 let routeFault: { readonly reason: string; readonly fault: 'harness_fault' | 'worker' } | null = null
701 for (;;) {
702 attempt = await smokeAttempt(config, opts, secrets, rows, harnessRows, requests)
703 if (attempt.restart === null) break
704 // The discarded attempt's repo saw a write that is not evidence; delete it like a passed one.
705 const bound = cleanupBound(config)
706 const removed = await confirmDelete(attempt.name, bound, { once: () => deleteOnce(attempt.name, config, secrets), sleep: wait })
707 await noteCleanup(removed, bound)
708 if (removed.kind === 'leak') leaks.push(removed.repo)
709 discarded.push(attempt.name)
710 if (restarts >= SMOKE_RESTARTS) break
711 restarts += 1
712 const route = await reconfirmRoute(config, opts.version, opts.pin)
713 if (!route.ok) {
714 routeFault = { reason: route.reason, fault: route.fault }
715 break
716 }
717 }
718 let detail = attempt.detail
719 let passed = attempt.passed
720 let harness = attempt.harness
721 let fault: HarnessFaultKind | null = attempt.fault
722 const name = attempt.name
723 let deleted = attempt.deleted
724 if (attempt.restart !== null) {
725 passed = false
726 harness = true
727 if (routeFault !== null) {
728 detail = `${attempt.restart}; ${routeFault.reason}`
729 fault = routeFault.fault === 'worker' ? null : 'harness_fault'
730 harness = routeFault.fault !== 'worker'
731 } else {
732 detail = fault === 'harness_unhealthy'
733 ? `harness unhealthy, prod untouched: ${attempt.restart}; ${restarts} restarts exhausted`
734 : `${attempt.restart}; ${restarts} restarts exhausted`
735 if (fault === null) fault = 'harness_fault'
736 }
737 deleted = true
738 }
739 try {
740 await writeText(opts.runDir, opts.pin ? 'smoke.csv' : 'live-smoke.csv', csvDocument(rows), secrets)
741 if (passed) {
742 const bound = cleanupBound(config)
743 const removed = await confirmDelete(name, bound, {
744 once: () => deleteOnce(name, config, secrets),
745 sleep: wait,
746 })
747 await noteCleanup(removed, bound)
748 switch (removed.kind) {
749 case 'deleted':
750 deleted = true
751 break
752 case 'leak':
753 deleted = false
754 leaks.push(removed.repo)
755 break
756 default:
757 assertNever(removed)
758 }
759 }
760 if (passed && leaks.length > 0) detail = `ok; cleanup_leak ${leaks.join(' ')}`
761 await appendCleanupFile(opts.runDir, leaks)
762 } catch (error) {
763 detail = redact(errorMessage(error), secrets)
764 passed = false
765 harness = true
766 }
767 const counted = rows.filter(row => !harnessRows.has(row))
768 const httpPass = counted.filter(row => row.pass).length
769 const httpFail = counted.length - httpPass
770 const passedCount = httpPass + (passed ? 1 : 0)
771 const failedCount = httpFail + (!passed && httpFail === 0 ? 1 : 0)
772 const harnessEvents = harnessRows.size
773 const report: SmokeReport = { version: opts.version, repo: name, deleted, cleanup_leak: leaks, requests, refs: attempt.refs, detail, passed: passedCount, failed: failedCount, harnessEvents, restarts, discarded }
774 await writeJson(opts.runDir, opts.pin ? 'smoke.json' : 'live-smoke.json', report, secrets)
775 const runLog = activeLog()
776 const outcomeFault = harness ? (fault === null ? 'harness_fault' : fault) : null
777 if (runLog) {
778 runLog.addSmoke(passedCount, failedCount)
779 const ref = await runLog.observation({
780 measurement: { source: 'lifecycle-smoke', passed: passedCount, failed: failedCount, harnessEvents, restarts, detail },
781 query: { source: 'lifecycle' },
782 from: smokeStarted,
783 to: Date.now(),
784 raw: JSON.stringify({ detail, requests: requests.length, repo: name, deleted, cleanup_leak: leaks, harnessEvents, restarts, discarded }),
785 })
786 const informationalSmoke = !passed && !harness && config.policy.gates.smoke === 'informational'
787 await runLog.decision({
788 inputs: [ref],
789 rules: [{ name: 'correctness.target', value: CORRECTNESS.target }],
790 outcome: passed || informationalSmoke ? 'advance' : outcomeFault !== null ? outcomeFault : 'stop',
791 reason: informationalSmoke ? `informational: ${detail}` : detail,
792 })
793 }
794 const smokeHard = config.policy.gates.smoke === 'hard'
795 if (!passed && (harness || smokeHard)) return fail('smoke', `${detail}; repo ${deleted ? 'deleted' : name}`, false, outcomeFault !== null ? outcomeFault : 'worker')
796 return pass('smoke', report)
Diagnostics4 git calls · 29 ms
CallStatusSizeTime029 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2008.5 KiB · hit24 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=24
git.response.bytes=8670
git.response.cache=HIT
git.response.duration_ms=24
git.fetch.wants=0619a8fde8ef35c29e3faa17d4833e7c0ae9d95f
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=8670
git.pkt.payload.max=8649
git.sideband.band1.bytes=8648
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e3373ccb5a24ce
Request
f191d008…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format