main
ply4 git calls338 ms44 KiB2 cachedprotocol v2
216 lines8.3 KiBRaw
1import { open, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'
2import { join } from 'node:path'
3import { mapGet } from '../../src/absent.ts'
4import { unwrapOr } from '../../src/option.ts'
5import { asRecord } from './json.ts'
6import { RUN_ID } from './run-dir.ts'
8// Measured 2026-11-07: a ramp run's directory was 120-135 MB (captures/ 73 MB, raw/ 53 MB) and about 50 runs filled
9// the disk twice. Nothing deleted old runs. Retention runs once at the start of every run that creates a directory.
11export type RunStatus = 'pass' | 'fail' | 'unfinished'
13/** A run entry as retention sees it. `compacted` runs keep only their top-level summary files. */
14export type RunEntry = {
15 readonly name: string
16 readonly status: RunStatus
17 readonly compacted: boolean
20export type RetentionPolicy = {
21 /** Newest passing runs kept whole (events, screenshots, summaries). */
22 readonly keepPass: number
23 /** Newest failed, rolled-back, or abandoned runs kept whole, blobs included. */
24 readonly keepFail: number
25 /** Further runs, newest first, that keep their summary files after compaction. Older ones are removed. */
26 readonly keepSummaries: number
27 /** An unfinished run younger than this is never touched: it may still be running or about to resume. */
28 readonly graceMs: number
31export const RETENTION: RetentionPolicy = {
32 keepPass: 10,
33 keepFail: 10,
34 keepSummaries: 500,
35 graceMs: 24 * 3600_000,
38export type RetentionPlan = {
39 readonly keep: readonly string[]
40 /** Whole runs that passed cleanly: their leftover blobs and pre-2026-11-07 captures/ and raw/ go. */
41 readonly strip: readonly string[]
42 readonly compact: readonly string[]
43 readonly remove: readonly string[]
46/** Inverse of `createRunDir`'s name. */
47export function runStartedAt(name: string): number {
48 const m = /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/.exec(name)
49 if (!m) return Number.NaN
50 return Date.parse(`${m[1]}T${m[2]}:${m[3]}:${m[4]}.${m[5]}Z`)
53/** Pure. Newest first by name; a failed or abandoned run counts against `keepFail`. */
54export function planRetention(entries: readonly RunEntry[], policy: RetentionPolicy, now: number, exclude: ReadonlySet<string> = new Set()): RetentionPlan {
55 const keep: string[] = []
56 const strip: string[] = []
57 const compact: string[] = []
58 const remove: string[] = []
59 let passes = 0
60 let fails = 0
61 let summaries = 0
62 const sorted = entries.filter(entry => RUN_ID.test(entry.name)).sort((a, b) => (a.name < b.name ? 1 : a.name > b.name ? -1 : 0))
63 for (const entry of sorted) {
64 if (exclude.has(entry.name)) {
65 keep.push(entry.name)
66 continue
67 }
68 if (entry.status === 'unfinished' && !entry.compacted && !(now - runStartedAt(entry.name) >= policy.graceMs)) {
69 keep.push(entry.name)
70 continue
71 }
72 if (!entry.compacted) {
73 if (entry.status === 'pass' && passes < policy.keepPass) {
74 passes += 1
75 strip.push(entry.name)
76 continue
77 }
78 if (entry.status !== 'pass' && fails < policy.keepFail) {
79 fails += 1
80 keep.push(entry.name)
81 continue
82 }
83 }
84 if (summaries < policy.keepSummaries) {
85 summaries += 1
86 if (entry.compacted) keep.push(entry.name)
87 else compact.push(entry.name)
88 continue
89 }
90 remove.push(entry.name)
91 }
92 return { keep, strip, compact, remove }
95const TAIL_BYTES = 256 * 1024
97async function tail(path: string, bytes: number): Promise<string> {
98 const handle = await open(path, 'r')
99 try {
100 const { size } = await handle.stat()
101 const length = Math.min(size, bytes)
102 const buffer = Buffer.alloc(length)
103 await handle.read(buffer, 0, length, size - length)
104 return buffer.toString('utf8')
105 } finally {
106 await handle.close()
107 }
110/** `run_finished` is the last event a run appends, so the tail of events.jsonl decides. A rollback counts as a failure. */
111export function statusFromEvents(text: string): RunStatus {
112 const lines = text.split('\n')
113 for (let i = lines.length - 1; i >= 0; i--) {
114 const line = lines[i]
115 if (!line || !line.includes('"run_finished"')) continue
116 try {
117 const event = asRecord(JSON.parse(line) as unknown)
118 const payload = event ? asRecord(event.payload) : null
119 if (!payload || payload.event !== 'run_finished') continue
120 return payload.verdict === 'pass' && payload.rolledBack !== true ? 'pass' : 'fail'
121 } catch {
122 continue
123 }
124 }
125 return 'unfinished'
128export const PRUNED = 'pruned.json'
130async function readEntry(root: string, name: string): Promise<RunEntry | null> {
131 const dir = join(root, name)
132 try {
133 const marker = asRecord(JSON.parse(await readFile(join(dir, PRUNED), 'utf8')) as unknown)
134 const status = marker ? marker.status : null
135 if (status === 'pass' || status === 'fail' || status === 'unfinished') return { name, status, compacted: true }
136 } catch {
137 // Not compacted.
138 }
139 try {
140 return { name, status: statusFromEvents(await tail(join(dir, 'events.jsonl'), TAIL_BYTES)), compacted: false }
141 } catch {
142 // No events.jsonl: a run that died before its first event. Abandoned.
143 return { name, status: 'unfinished', compacted: false }
144 }
147async function sizeOf(path: string): Promise<number> {
148 const info = await stat(path)
149 if (!info.isDirectory()) return info.size
150 let total = 0
151 for (const entry of await readdir(path)) total += await sizeOf(join(path, entry))
152 return total
155async function removeAll(dir: string, names: readonly string[]): Promise<{ removed: string[]; bytes: number }> {
156 const removed: string[] = []
157 let bytes = 0
158 for (const name of names) {
159 const path = join(dir, name)
160 try {
161 bytes += await sizeOf(path)
162 } catch {
163 continue
164 }
165 await rm(path, { recursive: true, force: true })
166 removed.push(name)
167 }
168 return { removed, bytes }
171/** Pre-2026-11-07 runs wrote plain `captures/` and `raw/`; current runs write gzipped `blobs/`. */
172const BULK_DIRS = ['blobs', 'captures', 'raw']
173/** Duplicated in the ledger, or a large per-file index that names files compaction removes. */
174const BULK_FILES = ['events.jsonl', 'manifest.json']
176/**
177 * Keep the top-level summary files (verdict, telemetry, smoke, visual, deploy, ramp: tens of KB) and drop
178 * every directory (blobs, captures, raw, visual screenshots) plus events.jsonl, whose lines are in the ledger.
179 */
180async function compactRun(dir: string, status: RunStatus, now: number): Promise<number> {
181 const entries = await readdir(dir, { withFileTypes: true })
182 const names = entries.filter(entry => entry.isDirectory() || BULK_FILES.includes(entry.name)).map(entry => entry.name)
183 const { removed, bytes } = await removeAll(dir, names)
184 const marker = { status, compacted_at: new Date(now).toISOString(), removed: removed.sort(), bytes_freed: bytes }
185 await writeFile(join(dir, PRUNED), JSON.stringify(marker, null, 2) + '\n')
186 return bytes
189export type RetentionReport = RetentionPlan & { readonly bytesFreed: number }
191/** Apply `planRetention` under `root`. `exclude` names directories that must not change (a run being resumed). */
192export async function pruneRuns(root: string, exclude: readonly string[] = [], policy: RetentionPolicy = RETENTION, now = Date.now()): Promise<RetentionReport> {
193 let names: string[]
194 try {
195 names = (await readdir(root, { withFileTypes: true })).filter(entry => entry.isDirectory() && RUN_ID.test(entry.name)).map(entry => entry.name)
196 } catch {
197 return { keep: [], strip: [], compact: [], remove: [], bytesFreed: 0 }
198 }
199 const entries: RunEntry[] = []
200 for (const name of names) {
201 const entry = await readEntry(root, name)
202 if (entry) entries.push(entry)
203 }
204 const plan = planRetention(entries, policy, now, new Set(exclude))
205 const status = new Map(entries.map(entry => [entry.name, entry.status]))
206 let bytesFreed = 0
207 for (const name of plan.strip) bytesFreed += (await removeAll(join(root, name), BULK_DIRS)).bytes
208 for (const name of plan.compact) bytesFreed += await compactRun(join(root, name), unwrapOr(mapGet(status, name), 'unfinished'), now)
209 for (const name of plan.remove) bytesFreed += (await removeAll(root, [name, `${name}.debrief`])).bytes
210 return { ...plan, bytesFreed }
213export function retentionLine(report: RetentionReport): string {
214 const mb = (report.bytesFreed / 1e6).toFixed(1)
215 return `retention: ${report.keep.length + report.strip.length} whole, ${report.compact.length} compacted, ${report.remove.length} removed, ${mb} MB freed`
Diagnostics4 git calls · 338 ms
CallStatusSizeTime0338 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=333
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2003.0 KiB333 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3079
git.response.cache=MISS
git.response.duration_ms=333
git.fetch.wants=f3b9a1214637af77296777da04fff84bd537a204
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3079
git.pkt.payload.max=3058
git.sideband.band1.bytes=3057
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e2f23929f7fda9
Request
227650a9…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format