main
1// Watched refs (src/refs-watch.ts, src/gitcache.ts wrefs): who grants the long policy, when a repo is purged (a change,
2// a first baseline, a gap, a failing repo, a fill that saw refs the watcher did not), the repeat purge, and GitCache's
3// side (key kind, policy, tag, lease timeout, seen report, the purge RPC).
4import { test } from 'node:test';
5import assert from 'node:assert/strict';
6import {
7 FAIL_PURGE_MS, HEALTHY_MS, LEASE_MS, LEASE_TIMEOUT_MS, SECOND_PURGE_MS, WATCHED_REFS_CACHE, WATCH_GAP_MS, WATCH_TICK_MS,
8 applyTick, emptyState, leaseRepo, needsAlarm, parseLsRefsAnswer, pollList, purgeFailed, refMapOf, refsTag, sawRefs,
9 type PollResult, type RefMap,
10} from '../src/refs-watch.ts';
11import {
12 H_FILLED, H_REFS_POLICY, REFS_CACHE, REFS_STATE_HEADER, createLoopbackFetch, keyHash, keyPath, purgeRefsTags, refsState, routedKind,
13 serveGitCache, type LoopbackBinding,
14} from '../src/gitcache.ts';
15import { ATTR_REFS_AGE, ATTR_REFS_POLICY, createRayLog, durationMs, observeUpstream, parseCfRay, stampUpstreamSpans } from '../src/rays.ts';
16import { PUBLIC_GIT_ORIGIN, PUBLIC_NAMESPACE, type TokenSource } from '../src/public.ts';
18const A = 'a'.repeat(40);
19const B = 'b'.repeat(40);
20const C = 'c'.repeat(40);
21const PUB = `${PUBLIC_GIT_ORIGIN}/git/${PUBLIC_NAMESPACE}/ply.git`;
22const GH = 'https://github.com/octocat/Hello-World.git';
23const pkt = (s: string) => `${(s.length + 4).toString(16).padStart(4, '0')}${s}`;
24const enc = (s: string) => new TextEncoder().encode(s);
25const LSREFS = enc(`${pkt('command=ls-refs\n')}0001${pkt('symrefs\n')}${pkt('ref-prefix HEAD\n')}0000`);
26const ok = (refs: RefMap): PollResult => ({ ok: true, refs });
27const main = (oid: string): RefMap => ({ HEAD: oid, 'HEAD->': 'refs/heads/main', 'refs/heads/main': oid });
29test('lease: a new repo is not granted until the watcher has polled it; then only while its last poll is recent', () => {
30 const s = emptyState();
31 const t = 1_000_000;
32 assert.deepEqual(leaseRepo(s, 'ply', t), { grant: false, reason: 'new' });
33 assert.deepEqual(leaseRepo(s, 'ply', t + 1), { grant: false, reason: 'unpolled' });
34 assert.deepEqual(leaseRepo(s, '../x', t), { grant: false, reason: 'bad_name' });
35 applyTick(s, { ply: ok(main(A)) }, t + 100);
36 assert.deepEqual(leaseRepo(s, 'ply', t + 200), { grant: true, reason: 'ok' });
37 assert.deepEqual(leaseRepo(s, 'ply', t + 100 + HEALTHY_MS + 1), { grant: false, reason: 'stale' });
38 applyTick(s, { ply: { ok: false, error: 'down' } }, t + 100 + HEALTHY_MS + 2);
39 assert.deepEqual(leaseRepo(s, 'ply', t + 100 + HEALTHY_MS + 3), { grant: false, reason: 'failing' });
40 assert.ok((s.repos['ply'] as { lease: number }).lease >= t + 100 + HEALTHY_MS + 3 + LEASE_MS - 1, 'each lease extends the watch');
41});
43test('tick: a first baseline and every change purge, and each purge repeats SECOND_PURGE_MS later; no change, no purge', () => {
44 const s = emptyState();
45 let t = 5_000_000;
46 leaseRepo(s, 'ply', t);
47 let r = applyTick(s, { ply: ok(main(A)) }, t);
48 assert.deepEqual(r.purge, ['ply'], 'an answer from an earlier lease may still be cached');
49 t += WATCH_TICK_MS;
50 r = applyTick(s, { ply: ok(main(A)) }, t);
51 assert.deepEqual(r.purge, []);
52 t += WATCH_TICK_MS;
53 r = applyTick(s, { ply: ok(main(B)) }, t);
54 assert.deepEqual(r.purge, ['ply']);
55 assert.deepEqual(r.changed, [{ repo: 'ply', from: A.slice(0, 12), to: B.slice(0, 12) }]);
56 const due = s.again['ply'] as number;
57 assert.equal(due, t + SECOND_PURGE_MS);
58 r = applyTick(s, { ply: ok(main(B)) }, t + WATCH_TICK_MS);
59 assert.deepEqual(r.purge, [], 'not due yet');
60 r = applyTick(s, { ply: ok(main(B)) }, due);
61 assert.deepEqual(r.purge, ['ply']);
62 assert.deepEqual(r.again, ['ply']);
63 assert.equal('ply' in s.again, false, 'a repeat is not repeated');
64 // a tag or branch moving is a change too, and so is HEAD retargeting
65 r = applyTick(s, { ply: ok({ ...main(B), 'refs/tags/v1': C }) }, due + WATCH_TICK_MS);
66 assert.deepEqual(r.purge, ['ply']);
67 r = applyTick(s, { ply: ok({ ...main(B), 'refs/tags/v1': C, 'HEAD->': 'refs/heads/dev' }) }, due + 2 * WATCH_TICK_MS);
68 assert.deepEqual(r.purge, ['ply']);
69});
71test('tick: a gap purges every leased repo; a failing repo is purged once after FAIL_PURGE_MS; an expired lease stops the watch', () => {
72 const s = emptyState();
73 let t = 9_000_000;
74 leaseRepo(s, 'ply', t);
75 leaseRepo(s, 'curl', t);
76 applyTick(s, { ply: ok(main(A)), curl: ok(main(B)) }, t);
77 t += WATCH_TICK_MS;
78 assert.deepEqual(applyTick(s, { ply: ok(main(A)), curl: ok(main(B)) }, t).purge, []);
79 t += WATCH_GAP_MS + 1;
80 const g = applyTick(s, { ply: ok(main(A)), curl: ok(main(B)) }, t);
81 assert.equal(g.gap, true);
82 assert.deepEqual(g.purge, ['curl', 'ply'], 'a push inside the gap (even one reverted) is not missed');
83 // failing
84 t += WATCH_TICK_MS;
85 assert.deepEqual(applyTick(s, { ply: { ok: false, error: 'x' }, curl: ok(main(B)) }, t).purge, []);
86 const failStart = t;
87 t = failStart + FAIL_PURGE_MS;
88 assert.deepEqual(applyTick(s, { ply: { ok: false, error: 'x' }, curl: ok(main(B)) }, t).purge.includes('ply'), true);
89 t += WATCH_TICK_MS;
90 assert.deepEqual(applyTick(s, { ply: { ok: false, error: 'x' }, curl: ok(main(B)) }, t).purge.includes('ply'), false, 'once per failure');
91 // recovered: a different head than before the failure is a change
92 t += WATCH_TICK_MS;
93 assert.deepEqual(applyTick(s, { ply: ok(main(C)), curl: ok(main(B)) }, t).changed.map(c => c.repo), ['ply']);
94 // a repo leased while a tick's polls were in flight has no poll: left alone, not failing
95 leaseRepo(s, 'deepfile', t);
96 applyTick(s, { ply: ok(main(C)), curl: ok(main(B)) }, t + 1);
97 assert.equal((s.repos['deepfile'] as { failingSince: number | null }).failingSince, null);
98 assert.deepEqual(pollList(s), ['curl', 'deepfile', 'ply']);
99 // expiry
100 const end = t + LEASE_MS + 1;
101 const e = applyTick(s, {}, end);
102 assert.deepEqual(e.expired.sort(), ['curl', 'deepfile', 'ply']);
103 assert.deepEqual(Object.keys(s.repos), []);
104 // repeats still outstanding keep the alarm; once they ran, it stops
105 assert.equal(needsAlarm(s), Object.keys(s.again).length > 0);
106 applyTick(s, {}, end + SECOND_PURGE_MS);
107 assert.equal(needsAlarm(s), false);
108});
110test('every tick polls every leased repo', () => {
111 const s = emptyState();
112 const t = 7_000_000;
113 leaseRepo(s, 'b', t);
114 leaseRepo(s, 'a', t);
115 for (let i = 1; i <= 8; i++) {
116 assert.deepEqual(pollList(s), ['a', 'b']);
117 applyTick(s, { a: ok(main(A)), b: ok(main(A)) }, t + i * WATCH_TICK_MS);
118 }
119});
121test('seen: a fill that stored refs the watcher has not seen purges at the next tick; a failed purge is retried', () => {
122 const s = emptyState();
123 const t = 2_000_000;
124 leaseRepo(s, 'ply', t);
125 assert.equal(sawRefs(s, 'ply', [['HEAD', A]]), false, 'no baseline yet: nothing to compare');
126 applyTick(s, { ply: ok(main(A)) }, t);
127 applyTick(s, { ply: ok(main(A)) }, t + WATCH_TICK_MS);
128 assert.equal(sawRefs(s, 'ply', [['HEAD', A], ['refs/heads/main', A], ['HEAD->', 'refs/heads/main']]), false);
129 assert.equal(sawRefs(s, 'ply', [['HEAD', B]]), true, 'a push and its revert inside one tick');
130 assert.equal(sawRefs(s, 'other', [['HEAD', B]]), false);
131 assert.equal(needsAlarm(s), true);
132 const r = applyTick(s, { ply: ok(main(A)) }, t + 2 * WATCH_TICK_MS);
133 assert.deepEqual(r.purge, ['ply']);
134 purgeFailed(s, r.purge);
135 assert.deepEqual(applyTick(s, { ply: ok(main(A)) }, t + 3 * WATCH_TICK_MS).purge, ['ply']);
136 // a watcher whose ls-refs carried no symref target does not count HEAD-> as a difference
137 const t2 = emptyState();
138 leaseRepo(t2, 'ply', t);
139 applyTick(t2, { ply: ok({ HEAD: A }) }, t);
140 assert.equal(sawRefs(t2, 'ply', [['HEAD', A], ['HEAD->', 'refs/heads/main']]), false);
141});
143test('parseLsRefsAnswer and refMapOf agree on the same listing', () => {
144 const body = enc(`${pkt(`${A} HEAD symref-target:refs/heads/main\n`)}${pkt(`${A} refs/heads/main\n`)}${pkt(`${B} refs/tags/v1 peeled:${C}\n`)}${pkt('unborn HEAD\n')}0000`);
145 const parsed = parseLsRefsAnswer(body);
146 assert.deepEqual(parsed, [['HEAD', A], ['HEAD->', 'refs/heads/main'], ['refs/heads/main', A], ['refs/tags/v1', B]]);
147 const map = refMapOf([{ name: 'HEAD', oid: A }, { name: 'refs/heads/main', oid: A }, { name: 'refs/tags/v1', oid: B }], 'refs/heads/main');
148 assert.deepEqual(Object.fromEntries(parsed), map);
149 assert.deepEqual(parseLsRefsAnswer(enc('zzzz')), []);
150 assert.deepEqual(parseLsRefsAnswer(enc('0000')), []);
151 assert.equal(refsTag('ply'), `ply-refs-${PUBLIC_NAMESPACE}-ply`);
152});
154test('routedKind: only a public-namespace ls-refs becomes wrefs, and only with the watched policy on', () => {
155 assert.equal(routedKind('refs', 'POST', `${PUB}/git-upload-pack`, { watchedRefs: true }), 'wrefs');
156 assert.equal(routedKind('refs', 'POST', `${PUB}/git-upload-pack`, { watchedRefs: false }), 'refs');
157 assert.equal(routedKind('refs', 'POST', `${GH}/git-upload-pack`, { watchedRefs: true }), 'refs', 'forges keep the 15 s policy');
158 assert.equal(routedKind('pack', 'POST', `${PUB}/git-upload-pack`, { watchedRefs: true }), 'pack');
159 assert.equal(routedKind(null, 'POST', `${PUB}/git-upload-pack`, { watchedRefs: true }), null);
160});
162test('createLoopbackFetch: a watched ls-refs uses the wrefs key and reports policy and age to the ray log and span', async () => {
163 const urls: string[] = [];
164 const filled = Date.now() - 42_000;
165 const binding: LoopbackBinding = {
166 async fetch(req) {
167 urls.push(new URL(req.url).pathname);
168 return new Response('refs', { status: 200, headers: { 'content-type': 'application/x-git-upload-pack-result', 'cf-cache-status': 'HIT', [H_REFS_POLICY]: 'watched;ok', [H_FILLED]: String(filled) } });
169 },
170 };
171 const rays = createRayLog();
172 const f = observeUpstream(createLoopbackFetch(async () => new Response('d'), () => binding, { watchedRefs: true }), rays);
173 const res = await f(`${PUB}/git-upload-pack`, { method: 'POST', body: LSREFS, headers: { 'git-protocol': 'version=2' } });
174 assert.equal(urls[0], keyPath('wrefs', await keyHash('wrefs', `${PUB}/git-upload-pack`, 'version=2', LSREFS)));
175 assert.match(String(res.headers.get(REFS_STATE_HEADER)), /^watched;ok;age=4[12]$/);
176 const span = { name: 'git.upload-pack ls-refs', startTimeUnixNano: '1', attrs: { 'http.request.method': 'POST' } as Record<string, unknown> };
177 stampUpstreamSpans([span], rays.report());
178 assert.equal(span.attrs[ATTR_REFS_POLICY], 'watched;ok');
179 assert.ok([41, 42].includes(span.attrs[ATTR_REFS_AGE] as number));
180 // off: the refs key, and the short policy reported
181 const g = createLoopbackFetch(async () => new Response('d'), () => binding);
182 const r2 = await g(`${PUB}/git-upload-pack`, { method: 'POST', body: LSREFS, headers: { 'git-protocol': 'version=2' } });
183 assert.match(String(urls[1]), /\/refs\//);
184 assert.match(String(r2.headers.get(REFS_STATE_HEADER)), /^short;ttl;age=/);
185 assert.equal(refsState('pack', new Headers(), 0), null);
186 assert.equal(refsState('wrefs', new Headers(), 0), 'none;age=-1');
187 void durationMs; void parseCfRay;
188});
190function tokens(): TokenSource {
191 const tok = `art_v1_${'c4c4'.repeat(10)}?expires=4102444800`;
192 return { readToken: async () => tok, secrets: () => [tok] };
193}
195async function wrefsRequest(url: string, body: Uint8Array) {
196 const hash = await keyHash('wrefs', url, 'version=2', body);
197 const headers = new Headers({ 'x-ply-gc-url': url, 'x-ply-gc-proto': 'version=2', 'x-ply-gc-ua': 'ply-test', 'x-ply-gc-body': btoa(String.fromCharCode(...body)) });
198 return new Request(`https://gitcache.internal${keyPath('wrefs', hash)}`, { headers });
199}
201function watchCtx(lease: (repo: string) => Promise<{ grant: boolean; reason: string }>) {
202 const seen: [string, [string, string][]][] = [];
203 const waited: Promise<unknown>[] = [];
204 const ns = {
205 idFromName: (n: string) => n,
206 get: () => ({ lease, seen: async (repo: string, refs: [string, string][]) => { seen.push([repo, refs]); return false; } }),
207 };
208 return { seen, waited, ctx: { exports: { RefsWatch: ns }, waitUntil: (p: Promise<unknown>) => { waited.push(p); } } };
209}
211const LISTING = `${pkt(`${A} HEAD symref-target:refs/heads/main\n`)}0000`;
212const up: typeof globalThis.fetch = async () => new Response(LISTING, { status: 200, headers: { 'content-type': 'application/x-git-upload-pack-result' } });
214test('serveGitCache wrefs: a granted lease stores the watched policy with the repo tag and reports the refs; anything else stores 5 s + 10 s', async () => {
215 const granted = watchCtx(async () => ({ grant: true, reason: 'ok' }));
216 const res = await serveGitCache(await wrefsRequest(`${PUB}/git-upload-pack`, LSREFS), tokens(), granted.ctx, up);
217 assert.equal(res.status, 200);
218 assert.equal(res.headers.get('cache-control'), WATCHED_REFS_CACHE);
219 assert.match(String(res.headers.get('cache-tag')), new RegExp(`,${refsTag('ply')}$`));
220 assert.equal(res.headers.get(H_REFS_POLICY), 'watched;ok');
221 assert.ok(Number(res.headers.get(H_FILLED)) > 0);
222 assert.equal(await res.text(), LISTING);
223 await Promise.all(granted.waited);
224 assert.deepEqual(granted.seen, [['ply', [['HEAD', A], ['HEAD->', 'refs/heads/main']]]]);
225 const refused = watchCtx(async () => ({ grant: false, reason: 'new' }));
226 const short = await serveGitCache(await wrefsRequest(`${PUB}/git-upload-pack`, LSREFS), tokens(), refused.ctx, up);
227 assert.equal(short.headers.get('cache-control'), REFS_CACHE);
228 assert.equal(short.headers.get(H_REFS_POLICY), 'short;new');
229 assert.match(String(short.headers.get('cache-tag')), new RegExp(refsTag('ply')), 'tagged even when short');
230 assert.deepEqual(refused.seen, []);
231 const broken = watchCtx(async () => { throw new Error('do down'); });
232 assert.equal((await serveGitCache(await wrefsRequest(`${PUB}/git-upload-pack`, LSREFS), tokens(), broken.ctx, up)).headers.get('cache-control'), REFS_CACHE);
233 const none = await serveGitCache(await wrefsRequest(`${PUB}/git-upload-pack`, LSREFS), tokens(), {}, up);
234 assert.equal(none.headers.get('cache-control'), REFS_CACHE, 'no RefsWatch: short');
235 // a wrefs key for a forge is not a watched exchange
236 assert.equal((await serveGitCache(await wrefsRequest(`${GH}/git-upload-pack`, LSREFS), tokens(), granted.ctx, up)).status, 403);
237});
239test('serveGitCache wrefs: a lease that does not answer within LEASE_TIMEOUT_MS stores the short policy', async () => {
240 const slow = watchCtx(() => new Promise(() => {}));
241 const t0 = Date.now();
242 const res = await serveGitCache(await wrefsRequest(`${PUB}/git-upload-pack`, LSREFS), tokens(), slow.ctx, up);
243 assert.equal(res.headers.get('cache-control'), REFS_CACHE);
244 assert.ok(Date.now() - t0 >= LEASE_TIMEOUT_MS - 50);
245});
247test('purgeRefsTags: one purge of every repo tag from the calling entrypoint; a refusal or a throw is reported, never thrown', async () => {
248 const calls: unknown[] = [];
249 const ctx = { cache: { purge: async (o: unknown) => { calls.push(o); return { success: true, errors: [] }; } } };
250 assert.deepEqual((await purgeRefsTags(ctx, ['ply', 'curl', '../bad'])).ok, true);
251 assert.deepEqual(calls, [{ tags: [refsTag('ply'), refsTag('curl')] }]);
252 assert.deepEqual(await purgeRefsTags(ctx, []), { ok: true, ms: 0, error: '' });
253 const limited = await purgeRefsTags({ cache: { purge: async () => ({ success: false, errors: [{ code: 1, message: 'rate limited' }] }) } }, ['ply']);
254 assert.equal(limited.ok, false);
255 assert.match(limited.error, /rate limited/);
256 assert.equal((await purgeRefsTags({ cache: { purge: async () => { throw new Error('boom'); } } }, ['ply'])).ok, false);
257 assert.equal((await purgeRefsTags({}, ['ply'])).ok, false);
258 assert.equal((await purgeRefsTags(ctx, Array.from({ length: 101 }, (_, i) => `r${i}`))).ok, false);
259});
Diagnostics4 git calls · 29 ms
CallStatusSizeTime029 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2004.5 KiB · hit24 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=24 git.response.bytes=4644 git.response.cache=HIT git.response.duration_ms=24 git.fetch.wants=a72e19ffbcf4563ae3ed976b2b36282fee8875e8 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4644 git.pkt.payload.max=4623 git.sideband.band1.bytes=4622 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2f4e8e95c7089
- Request
- e70fbf18…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format