main
1// Watched refs: a long-lived refs cache for the public namespace that stays exactly as fresh as the short one, because
2// it is purged when a ref moves instead of expiring on a timer.
3//
4// Today every ls-refs answer is cached `max-age=5, stale-while-revalidate=10`: a page can show a head up to ~15 s old,
5// and a visitor on a quiet colo (nothing there for 15 s) waits on the upstream ls-refs (78% of sparse /r/ply wall time,
6// run sparse-ply on 2a4741c0). Here, for public-namespace repos only (forges keep the 15 s policy), GitCache stores the
7// answer for WATCHED_MAX_AGE_S + WATCHED_SWR_S, tagged with the repo (refsTag), and RefsWatch purges that tag when the
8// repo's refs change. Workers Cache purges are global (both tiers, every colo): in 10 trials × 9 regions every colo
9// served the new value 21-425 ms after `ctx.cache.purge` returned, p50 144 ms (lab_notebook ply-web-refs-2026-11-07).
10//
11// RefsWatch (one Durable Object) is the only thing that grants the long policy:
12// lease(repo) GitCache asks before it stores a watched answer, in parallel with its upstream ls-refs. The answer is
13// long-lived only when the watcher has polled that repo successfully within HEALTHY_MS; otherwise
14// GitCache stores today's short policy. The lease keeps the repo polled for LEASE_MS, longer than any
15// answer granted under it can be served (two cache tiers, each max-age + swr).
16// alarm every WATCH_TICK_MS: ls-refs each leased repo (straight upstream, never through the cache) and purge
17// every repo whose refs differ from the last poll. Every purge is repeated SECOND_PURGE_MS later: a
18// fill that read the old refs before the push and was stored after the first purge is gone by then
19// (GitCache aborts a watched upstream after REFS_FILL_TIMEOUT_MS < SECOND_PURGE_MS).
20// seen(repo, …) GitCache reports the refs it stored. A ref the watcher has not seen (a push and its revert inside one
21// tick, or a fill racing a push) purges the repo.
22// poke(repos) the mirror cron after it pushes: poll now instead of at the next tick.
23// ensure() the minute cron: an alarm that should be set but is not is set for now.
24//
25// Every tick polls every leased repo (Ivar, 2026-11-08: "Assume billing is not a factor"; a 12 s poll in 88cc23b2 took the
26// median push-to-visible from 3.6 s to 6.7 s, run fab3). A push costs two purge calls, the first and its repeat, against the Workers Cache
27// tag-purge limit of 5/min with a burst of 25; a refused purge is retried every tick.
28// Bound on how long a page can show an old head after any push (the mirror's, or one made straight to Artifacts):
29// WATCH_TICK_MS + one upstream ls-refs + purge propagation (≈ 3 s + 0.1-0.7 s + 0.1-0.4 s), against ~15 s before.
30// Failure modes, each closed: a tick later than WATCH_GAP_MS (alarm stalled, DO restarted, a deploy without the watcher)
31// purges every leased repo; a repo the watcher cannot read for FAIL_PURGE_MS is purged and no longer granted; a failed
32// purge is retried every tick; GitCache stores the short policy whenever the lease does not answer in LEASE_TIMEOUT_MS.
33// What is left: the watcher and its cron both stopped for longer than a tick while an answer granted before is still
34// cached. That answer can then be served for at most WATCHED_MAX_AGE_S + WATCHED_SWR_S (×2 across tiers).
35import { captureGroup, recordGet } from './absent.ts';
36import { PUBLIC_NAMESPACE, isPublicRepoName } from './public.ts';
38export const WATCHED_MAX_AGE_S = 300;
39export const WATCHED_SWR_S = 300;
40export const WATCHED_REFS_CACHE = `public, max-age=${WATCHED_MAX_AGE_S}, stale-while-revalidate=${WATCHED_SWR_S}`;
42/** The alarm's period: one ls-refs per leased repo per tick. */
43export const WATCH_TICK_MS = 3_000;
44/** One poll's own deadline. */
45export const POLL_TIMEOUT_MS = 5_000;
46/** A repo polled less recently than this is not granted the long policy. */
47export const HEALTHY_MS = 3 * WATCH_TICK_MS;
48/** A tick this late may have missed a push: every leased repo is purged. */
49export const WATCH_GAP_MS = 4 * WATCH_TICK_MS;
50/** The upstream ls-refs of a watched fill is aborted after this, so no fill outlives the second purge. */
51export const REFS_FILL_TIMEOUT_MS = 8_000;
52export const SECOND_PURGE_MS = REFS_FILL_TIMEOUT_MS + 2_000;
53/** A repo the watcher has failed to read for this long is purged once (fail closed) and is not granted until it reads. */
54export const FAIL_PURGE_MS = 2 * WATCH_TICK_MS;
55/** GitCache waits this long for lease(); no answer stores the short policy. */
56export const LEASE_TIMEOUT_MS = 2_000;
57/** Longer than a granted answer can be served: two tiers, each max-age + swr, plus a minute. */
58export const LEASE_MS = 2 * (WATCHED_MAX_AGE_S + WATCHED_SWR_S) * 1000 + 60_000;
59/** Workers Cache takes at most 100 tags per purge (developers.cloudflare.com/cache/how-to/purge-cache, 2026-11-07). */
60export const MAX_PURGE_TAGS = 100;
61/** The one RefsWatch object. */
62export const WATCH_NAME = `refs-${PUBLIC_NAMESPACE}`;
64/** The Cache-Tag on every watched refs answer of `repo`; RefsWatch purges it. Tags compare case-insensitively. */
65export function refsTag(repo: string): string {
66 return `ply-refs-${PUBLIC_NAMESPACE}-${repo}`;
67}
69/** name → oid. HEAD's symref target is kept as `HEAD->` so a retarget is a change too. */
70export type RefMap = Record<string, string>;
72export type RepoWatch = {
73 /** Poll this repo until then. */
74 lease: number;
75 /** The last successful poll's refs; null before the first. */
76 refs: RefMap | null;
77 polledAt: number;
78 failingSince: number | null;
79 failPurged: boolean;
80};
82export type WatchState = {
83 repos: Record<string, RepoWatch>;
84 /** The last tick that ran; 0 before the first. */
85 tickAt: number;
86 /** ls-refs sent since the minute cron last read this. */
87 polls: number;
88 /** repo → when its repeat purge is due. */
89 again: Record<string, number>;
90 /** Repos to purge at the next tick (a seen() mismatch, a failed purge). */
91 soon: string[];
92};
94export function emptyState(): WatchState {
95 return { repos: {}, tickAt: 0, polls: 0, again: {}, soon: [] };
96}
98export type LeaseAnswer = { grant: boolean; reason: 'ok' | 'new' | 'unpolled' | 'stale' | 'failing' | 'bad_name' };
100/** GitCache's question before it stores a watched answer. Starts watching a repo it does not know. */
101export function leaseRepo(state: WatchState, repo: string, now: number): LeaseAnswer {
102 if (!isPublicRepoName(repo)) return { grant: false, reason: 'bad_name' };
103 const found = recordGet(state.repos, repo);
104 if (found.tag === 'none') {
105 state.repos[repo] = { lease: now + LEASE_MS, refs: null, polledAt: 0, failingSince: null, failPurged: false };
106 return { grant: false, reason: 'new' };
107 }
108 const known = found.value;
109 known.lease = Math.max(known.lease, now + LEASE_MS);
110 if (known.refs === null) return { grant: false, reason: 'unpolled' };
111 if (known.failingSince !== null) return { grant: false, reason: 'failing' };
112 if (now - known.polledAt > HEALTHY_MS) return { grant: false, reason: 'stale' };
113 return { grant: true, reason: 'ok' };
114}
116/** GitCache stored these refs for `repo`. True when one differs from the watcher's view (the repo is purged next tick). */
117export function sawRefs(state: WatchState, repo: string, seen: readonly (readonly [string, string])[]): boolean {
118 const known = recordGet(state.repos, repo);
119 if (known.tag === 'none' || known.value.refs === null) return false;
120 const refs = known.value.refs;
121 // A HEAD target the watcher did not list (its ls-refs carried no symref) is not a difference.
122 const differs = seen.some(([name, oid]) => !(name === 'HEAD->' && !(name in refs)) && refs[name] !== oid);
123 if (differs && !state.soon.includes(repo)) state.soon.push(repo);
124 return differs;
125}
127export type PollResult = { ok: true; refs: RefMap } | { ok: false; error: string };
129export type TickReport = {
130 /** Repos to purge now (deduplicated, sorted). */
131 purge: string[];
132 changed: { repo: string; from: string; to: string }[];
133 gap: boolean;
134 /** ms since the previous tick, or null on the first. */
135 sinceMs: number | null;
136 failed: string[];
137 expired: string[];
138 again: string[];
139};
141function sameRefs(a: RefMap, b: RefMap): boolean {
142 const ak = Object.keys(a);
143 if (ak.length !== Object.keys(b).length) return false;
144 return ak.every(k => a[k] === b[k]);
145}
147/** A short label for a log line: HEAD's oid when there is one. */
148export function headLabel(refs: RefMap | null): string {
149 if (refs === null) return '';
150 const head = recordGet(refs, 'HEAD');
151 return head.tag === 'none' ? `${Object.keys(refs).length} refs` : head.value.slice(0, 12);
152}
154/** The repos to poll this tick: every leased one. */
155export function pollList(state: WatchState): string[] {
156 return Object.keys(state.repos).sort();
157}
159/**
160 * Apply one tick's polls. Mutates `state`. Every purge it asks for is also scheduled again SECOND_PURGE_MS later.
161 * `polls` holds a result for each repo this tick polled; a repo leased since has none and is left for the next tick.
162 */
163export function applyTick(state: WatchState, polls: Record<string, PollResult>, now: number): TickReport {
164 const purge = new Set<string>();
165 const changed: TickReport['changed'] = [];
166 const failed: string[] = [];
167 const expired: string[] = [];
168 const sinceMs = state.tickAt === 0 ? null : now - state.tickAt;
169 const gap = sinceMs !== null && sinceMs > WATCH_GAP_MS;
170 for (const repo of pollList(state)) {
171 const watch = state.repos[repo] as RepoWatch;
172 if (gap) purge.add(repo);
173 const result = recordGet(polls, repo);
174 const polled = result.tag === 'some' ? result.value : null;
175 if (polled === null) {
176 // Leased while this tick's polls were in flight: polled next tick.
177 } else if (polled.ok) {
178 // A first baseline purges too: an answer from an earlier lease of this repo may still be cached.
179 if (watch.refs === null || !sameRefs(watch.refs, polled.refs)) {
180 purge.add(repo);
181 changed.push({ repo, from: headLabel(watch.refs), to: headLabel(polled.refs) });
182 }
183 watch.refs = polled.refs;
184 watch.polledAt = now;
185 watch.failingSince = null;
186 watch.failPurged = false;
187 } else {
188 failed.push(repo);
189 if (watch.failingSince === null) watch.failingSince = now;
190 if (!watch.failPurged && now - watch.failingSince >= FAIL_PURGE_MS) {
191 purge.add(repo);
192 watch.failPurged = true;
193 }
194 }
195 if (now > watch.lease) {
196 expired.push(repo);
197 delete state.repos[repo];
198 }
199 }
200 for (const repo of state.soon) purge.add(repo);
201 state.soon = [];
202 const again: string[] = [];
203 for (const [repo, due] of Object.entries(state.again)) {
204 if (due <= now) {
205 again.push(repo);
206 purge.add(repo);
207 delete state.again[repo];
208 }
209 }
210 // Every purge except a repeat is repeated once (a repeat that coincides with a fresh purge is rescheduled).
211 for (const repo of purge) {
212 if (!again.includes(repo) || changed.some(c => c.repo === repo)) state.again[repo] = now + SECOND_PURGE_MS;
213 }
214 state.tickAt = now;
215 return { purge: [...purge].sort(), changed, gap, sinceMs, failed, expired, again };
216}
218/** A purge that did not go through: its repos are purged at the next tick. */
219export function purgeFailed(state: WatchState, repos: readonly string[]): void {
220 for (const repo of repos) if (!state.soon.includes(repo)) state.soon.push(repo);
221}
223/** Keep polling while a lease or a repeat purge is outstanding. */
224export function needsAlarm(state: WatchState): boolean {
225 return Object.keys(state.repos).length > 0 || Object.keys(state.again).length > 0 || state.soon.length > 0;
226}
228/**
229 * The refs in an ls-refs (protocol v2) answer: pkt-lines `<oid> <name>[ attr…]`. A `symref-target:` on HEAD is kept as
230 * `HEAD->`. Anything else (flush, unborn, a malformed line) is skipped: the result is only ever compared.
231 */
232export function parseLsRefsAnswer(bytes: Uint8Array): [string, string][] {
233 const out: [string, string][] = [];
234 let i = 0;
235 const dec = new TextDecoder();
236 while (i + 4 <= bytes.length) {
237 const len = Number.parseInt(dec.decode(bytes.subarray(i, i + 4)), 16);
238 if (!Number.isFinite(len)) break;
239 if (len < 4) { i += 4; continue; }
240 const line = dec.decode(bytes.subarray(i + 4, i + len)).replace(/\n$/, '');
241 i += len;
242 const m = /^([0-9a-f]{40}|[0-9a-f]{64}) (\S+)(.*)$/.exec(line);
243 if (m === null) continue;
244 const [, oid, name, rest] = m as unknown as [string, string, string, string];
245 out.push([name, oid]);
246 const target = captureGroup(/(^| )symref-target:(\S+)/.exec(rest), 2);
247 if (name === 'HEAD' && target.tag === 'some') out.push(['HEAD->', target.value]);
248 }
249 return out;
250}
252/** The watcher's RefMap from ply's ls-refs result. */
253export function refMapOf(refs: Iterable<{ name: string; oid: string }>, head: string | null): RefMap {
254 const out: RefMap = {};
255 for (const r of refs) out[r.name] = r.oid;
256 if (head !== null) out['HEAD->'] = head;
257 return out;
258}
Diagnostics4 git calls · 26 ms
CallStatusSizeTime026 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2004.8 KiB · hit21 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=21 git.response.bytes=4905 git.response.cache=HIT git.response.duration_ms=21 git.fetch.wants=4f94b459cc883eaa4a1efc3fa17461614846377a git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4905 git.pkt.payload.max=4884 git.sideband.band1.bytes=4883 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2f7fc8fa63a24
- Request
- 2ddd78a0…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format