main
1// Scratch-repo delete is lossy the same way a probe is. Run 2026-11-01T17-29-43-122Z
2// (web 9558b10) passed, then `cf artifacts namespaces repos delete` took 103s and
3// returned APIError "timeout", exit 1. The retry was 404 [10200] Repository not found:
4// the delete had succeeded. A timeout is not a failed release.
5import { readFile } from 'node:fs/promises'
6import { join } from 'node:path'
7import { assertNever } from '../../src/never.ts'
8import { asRecord } from './json.ts'
9import type { DecisionOutcome, ObservationRef } from './log.ts'
10import { writeJson } from './run-dir.ts'
12export const CLEANUP_FILE = 'cleanup.json'
14/** One `cf repos delete` invocation, timed by the caller. */
15export type DeleteCommand = {
16 readonly code: number
17 readonly stdout: string
18 readonly stderr: string
19 readonly latencyMs: number
20}
22/** What that invocation proved. `absent` is a 404: the repo is already gone. */
23export type DeleteAttempt =
24 | { readonly kind: 'deleted'; readonly latencyMs: number }
25 | { readonly kind: 'absent'; readonly latencyMs: number }
26 | { readonly kind: 'transient'; readonly latencyMs: number; readonly detail: string }
28/** `attempts` is the bound. The wait before retry n is `backoffMs * 2^(n-1)`. */
29export type CleanupBound = {
30 readonly attempts: number
31 readonly backoffMs: number
32}
34export type CleanupResult =
35 | { readonly kind: 'deleted'; readonly repo: string; readonly attempts: readonly DeleteAttempt[] }
36 | { readonly kind: 'leak'; readonly repo: string; readonly attempts: readonly DeleteAttempt[] }
38export type CleanupStep =
39 | { readonly kind: 'call' }
40 | { readonly kind: 'wait'; readonly waitMs: number }
41 | { readonly kind: 'done'; readonly result: CleanupResult }
43export type ListedRepo = {
44 readonly name: string
45 readonly createdAtMs: number | null
46}
48export type SweepReport = {
49 readonly deleted: readonly string[]
50 readonly leaks: readonly string[]
51}
53export type CleanupIo = {
54 readonly once: () => Promise<DeleteCommand>
55 readonly sleep: (ms: number) => Promise<void>
56}
58export type SweepInput = {
59 readonly listed: readonly ListedRepo[]
60 readonly nowMs: number
61 readonly staleMs: number
62 readonly prefix: string
63 readonly bound: CleanupBound
64 readonly once: (name: string) => Promise<DeleteCommand>
65 readonly sleep: (ms: number) => Promise<void>
66 readonly note: (result: CleanupResult) => Promise<void>
67}
69type CleanupLog = {
70 observation(input: {
71 readonly measurement: unknown
72 readonly query: unknown
73 readonly from: number
74 readonly to: number
75 readonly raw: string
76 }): Promise<ObservationRef>
77 decision(input: {
78 readonly inputs: readonly [ObservationRef, ...ObservationRef[]]
79 readonly rules: readonly [{ readonly name: string; readonly value: number | string }, ...{ readonly name: string; readonly value: number | string }[]]
80 readonly outcome: DecisionOutcome
81 readonly reason: string
82 }): Promise<void>
83}
85function clip(text: string, code: number): string {
86 const flat = text.replace(/\s+/g, ' ').trim()
87 if (flat.length === 0) return `exit ${code}`
88 return flat.slice(0, 240)
89}
91function isAbsent(text: string): boolean {
92 if (/repository not found/i.test(text)) return true
93 if (text.includes('[10200]')) return true
94 return /\b404\b/.test(text)
95}
97/** Exit 0 is deleted. 404, [10200], or "Repository not found" is deleted too. Anything else is retried. */
98export function classifyDelete(command: DeleteCommand): DeleteAttempt {
99 if (command.code === 0) return { kind: 'deleted', latencyMs: command.latencyMs }
100 const text = `${command.stdout}\n${command.stderr}`
101 if (isAbsent(text)) return { kind: 'absent', latencyMs: command.latencyMs }
102 return { kind: 'transient', latencyMs: command.latencyMs, detail: clip(text, command.code) }
103}
105function confirmed(attempts: readonly DeleteAttempt[]): boolean {
106 for (const attempt of attempts) {
107 switch (attempt.kind) {
108 case 'deleted':
109 case 'absent':
110 return true
111 case 'transient':
112 break
113 default:
114 return assertNever(attempt)
115 }
116 }
117 return false
118}
120/**
121 * Stop when a delete is confirmed, or when `bound.attempts` is exhausted.
122 * Exhaustion is a leak: the release already passed, and the repo name is the record.
123 */
124export function cleanupStep(repo: string, attempts: readonly DeleteAttempt[], bound: CleanupBound): CleanupStep {
125 if (confirmed(attempts)) return { kind: 'done', result: { kind: 'deleted', repo, attempts } }
126 if (attempts.length >= bound.attempts) return { kind: 'done', result: { kind: 'leak', repo, attempts } }
127 if (attempts.length === 0) return { kind: 'call' }
128 return { kind: 'wait', waitMs: bound.backoffMs * 2 ** (attempts.length - 1) }
129}
131export async function confirmDelete(repo: string, bound: CleanupBound, io: CleanupIo): Promise<CleanupResult> {
132 const attempts: DeleteAttempt[] = []
133 for (;;) {
134 const step = cleanupStep(repo, attempts, bound)
135 switch (step.kind) {
136 case 'done':
137 return step.result
138 case 'wait':
139 await io.sleep(step.waitMs)
140 attempts.push(classifyDelete(await io.once()))
141 break
142 case 'call':
143 attempts.push(classifyDelete(await io.once()))
144 break
145 default:
146 return assertNever(step)
147 }
148 }
149}
151/** Names whose age is strictly greater than `staleMs`. A missing timestamp is kept. */
152export function leakedRepos(repos: readonly ListedRepo[], nowMs: number, staleMs: number, prefix: string): readonly string[] {
153 const names: string[] = []
154 for (const repo of repos) {
155 if (!repo.name.startsWith(prefix)) continue
156 if (repo.createdAtMs === null) continue
157 if (nowMs - repo.createdAtMs <= staleMs) continue
158 names.push(repo.name)
159 }
160 return names
161}
163export function reposFromList(value: unknown): readonly ListedRepo[] {
164 if (!Array.isArray(value)) return []
165 const repos: ListedRepo[] = []
166 for (const item of value) {
167 const rec = asRecord(item)
168 if (!rec) continue
169 if (typeof rec.name !== 'string' || rec.name.length === 0) continue
170 const created = typeof rec.created_at === 'string' ? Date.parse(rec.created_at) : Number.NaN
171 repos.push({ name: rec.name, createdAtMs: Number.isFinite(created) ? created : null })
172 }
173 return repos
174}
176export async function sweepLeaks(input: SweepInput): Promise<SweepReport> {
177 const deleted: string[] = []
178 const leaks: string[] = []
179 for (const name of leakedRepos(input.listed, input.nowMs, input.staleMs, input.prefix)) {
180 const result = await confirmDelete(name, input.bound, {
181 once: () => input.once(name),
182 sleep: input.sleep,
183 })
184 await input.note(result)
185 switch (result.kind) {
186 case 'deleted':
187 deleted.push(result.repo)
188 break
189 case 'leak':
190 leaks.push(result.repo)
191 break
192 default:
193 return assertNever(result)
194 }
195 }
196 return { deleted, leaks }
197}
199export function mergeLeakNames(prior: readonly string[], extra: readonly string[]): readonly string[] {
200 const names: string[] = []
201 for (const name of prior) {
202 if (!names.includes(name)) names.push(name)
203 }
204 for (const name of extra) {
205 if (!names.includes(name)) names.push(name)
206 }
207 return names
208}
210export function cleanupLeakNames(value: unknown): readonly string[] {
211 const rec = asRecord(value)
212 if (!rec || !Array.isArray(rec.leaks)) return []
213 const names: string[] = []
214 for (const name of rec.leaks) {
215 if (typeof name === 'string' && name.length > 0 && !names.includes(name)) names.push(name)
216 }
217 return names
218}
220export async function appendCleanupFile(runDir: string, leaks: readonly string[]): Promise<void> {
221 if (leaks.length === 0) return
222 let prior: readonly string[] = []
223 try {
224 prior = cleanupLeakNames(JSON.parse(await readFile(join(runDir, CLEANUP_FILE), 'utf8')) as unknown)
225 } catch {
226 prior = []
227 }
228 await writeJson(runDir, CLEANUP_FILE, { leaks: mergeLeakNames(prior, leaks) })
229}
231function attemptDetail(attempt: DeleteAttempt): string {
232 switch (attempt.kind) {
233 case 'transient':
234 return attempt.detail
235 case 'deleted':
236 case 'absent':
237 return ''
238 default:
239 return assertNever(attempt)
240 }
241}
243function attemptText(attempt: DeleteAttempt): string {
244 switch (attempt.kind) {
245 case 'deleted':
246 return `${attempt.latencyMs}ms deleted`
247 case 'absent':
248 return `${attempt.latencyMs}ms absent`
249 case 'transient':
250 return `${attempt.latencyMs}ms transient`
251 default:
252 return assertNever(attempt)
253 }
254}
256function reasonOf(result: CleanupResult): string {
257 const parts = result.attempts.map(attemptText).join(', ')
258 switch (result.kind) {
259 case 'deleted':
260 return `deleted ${result.repo} (${parts})`
261 case 'leak':
262 return `cleanup_leak ${result.repo} (${parts})`
263 default:
264 return assertNever(result)
265 }
266}
268function decisionOutcome(result: CleanupResult): DecisionOutcome {
269 switch (result.kind) {
270 case 'deleted':
271 return 'advance'
272 case 'leak':
273 return 'cleanup_leak'
274 default:
275 return assertNever(result)
276 }
277}
279function nonempty(refs: readonly ObservationRef[]): [ObservationRef, ...ObservationRef[]] {
280 const first = refs[0]
281 if (!first) throw new Error('cleanup record needs an observation')
282 return [first, ...refs.slice(1)]
283}
285/** Ledger record. Each attempt's latency is its own observation. A leak names the repo. */
286export async function recordCleanup(log: CleanupLog, result: CleanupResult, bound: CleanupBound): Promise<void> {
287 const refs: ObservationRef[] = []
288 let ordinal = 0
289 for (const attempt of result.attempts) {
290 ordinal += 1
291 const to = Date.now()
292 const from = to - attempt.latencyMs
293 const measurement = {
294 source: 'repo-delete',
295 repo: result.repo,
296 attempt: ordinal,
297 kind: attempt.kind,
298 latencyMs: attempt.latencyMs,
299 detail: attemptDetail(attempt),
300 }
301 refs.push(await log.observation({
302 measurement,
303 query: { source: 'repo-delete', repo: result.repo },
304 from,
305 to,
306 raw: JSON.stringify(measurement),
307 }))
308 }
309 if (result.kind === 'leak') {
310 const now = Date.now()
311 const measurement = { source: 'cleanup-leak', repo: result.repo }
312 refs.push(await log.observation({
313 measurement,
314 query: { source: 'cleanup-leak', repo: result.repo },
315 from: now,
316 to: now,
317 raw: JSON.stringify(measurement),
318 }))
319 }
320 await log.decision({
321 inputs: nonempty(refs),
322 rules: [
323 { name: 'cleanup.attempts', value: bound.attempts },
324 { name: 'cleanup.backoffMs', value: bound.backoffMs },
325 ],
326 outcome: decisionOutcome(result),
327 reason: reasonOf(result),
328 })
329}
Diagnostics4 git calls · 31 ms
CallStatusSizeTime031 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=4 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2003.0 KiB · hit24 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=24 git.response.bytes=3048 git.response.cache=HIT git.response.duration_ms=24 git.fetch.wants=7a8caab34b01befe62b476b5927844fce4d7ff73 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3048 git.pkt.payload.max=3027 git.sideband.band1.bytes=3026 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2e30f9da64af4
- Request
- 7e354ab6…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format