main
ply4 git calls26 ms43 KiB3 cachedprotocol v2
169 lines7.2 KiBRaw
1// RefsWatch, the Durable Object that grants and revokes the long-lived refs policy of the public namespace (the policy and
2// its bounds: src/refs-watch.ts). Reached as ctx.exports.RefsWatch from GitCache (lease, seen) and from the cron (ensure,
3// poke); it purges through ctx.exports.GitCache.purgeRefs, because a Workers Cache tag is scoped to the entrypoint that
4// stored it. One object (WATCH_NAME); its whole state is one small JSON value.
5import { DurableObject } from 'cloudflare:workers';
6import { Tracer, arrayExporter, createTransport, lsRefs } from 'ply';
7import { chainGet, dig, fromNullable, recordGet } from './absent.ts';
8import { MIRROR_LIMITS } from './limits.ts';
9import { createTokenSource, publicRemote, scrubSecrets, type ArtifactsNamespace, type TokenSource } from './public.ts';
10import { checkRemote, createGuardedFetch, redirectOptions } from './remote.ts';
11import {
12 POLL_TIMEOUT_MS, WATCH_TICK_MS, applyTick, emptyState, leaseRepo, needsAlarm, pollList, purgeFailed, refMapOf, sawRefs,
13 type LeaseAnswer, type PollResult, type WatchState,
14} from './refs-watch.ts';
16type Env = { PUBLIC_REPOS: unknown };
17type PurgeAnswer = { ok: boolean; ms: number; error: string };
18type GitCacheRpc = { purgeRefs(repos: string[]): Promise<PurgeAnswer> };
20const STATE_KEY = 'state';
22function errorText(e: unknown): string {
23 const code = chainGet(e, 'code');
24 const message = chainGet(e, 'message');
25 return `${code.tag === 'some' ? String(code.value) : ''} ${message.tag === 'some' ? String(message.value) : String(e)}`.trim().slice(0, 160);
28/** One ls-refs of every ref, straight upstream with the binding's read token. Never through GitCache. */
29export async function pollRefs(repo: string, tokens: TokenSource, fetchImpl: typeof fetch = (i, n) => fetch(i, n)): Promise<PollResult> {
30 try {
31 const dst = checkRemote(publicRemote(repo));
32 const bearer = await tokens.readToken(repo);
33 const made = createTransport(dst.href, {
34 tracer: new Tracer({ exporter: arrayExporter(), resource: { 'service.name': 'ply-refswatch' } }),
35 limits: MIRROR_LIMITS, quirks: {}, auth: { bearer }, signal: AbortSignal.timeout(POLL_TIMEOUT_MS),
36 fetchImpl: createGuardedFetch(fetchImpl, dst), ...redirectOptions(dst),
37 });
38 if (!made.ok) return { ok: false, error: errorText(made.error) };
39 const listed = await lsRefs(made.value);
40 if (!listed.ok) return { ok: false, error: errorText(listed.error) };
41 return { ok: true, refs: refMapOf(listed.value.refs.values(), listed.value.head) };
42 } catch (e) {
43 return { ok: false, error: errorText(e) };
44 }
47function pollError(polls: Record<string, PollResult>, repo: string): string {
48 const found = recordGet(polls, repo);
49 return found.tag === 'some' && !found.value.ok ? found.value.error : '';
52export class RefsWatch extends DurableObject<Env> {
53 #state: WatchState | null = null;
54 #tokens: TokenSource | null = null;
55 #ticking = false;
57 async #load(): Promise<WatchState> {
58 if (this.#state === null) {
59 const stored = fromNullable(await this.ctx.storage.get<WatchState>(STATE_KEY));
60 this.#state = stored.tag === 'some' ? stored.value as WatchState : emptyState();
61 }
62 return this.#state;
63 }
65 async #save(): Promise<void> {
66 if (this.#state !== null) await this.ctx.storage.put(STATE_KEY, this.#state);
67 }
69 async #arm(at: number): Promise<void> {
70 const current = await this.ctx.storage.getAlarm();
71 if (current === null || current > at) await this.ctx.storage.setAlarm(at);
72 }
74 #tokenSource(): TokenSource | null {
75 if (this.#tokens === null && this.env.PUBLIC_REPOS) this.#tokens = createTokenSource(this.env.PUBLIC_REPOS as ArtifactsNamespace);
76 return this.#tokens;
77 }
79 async lease(repo: string): Promise<LeaseAnswer> {
80 const state = await this.#load();
81 const answer = leaseRepo(state, repo, Date.now());
82 await this.#save();
83 // A new repo is polled at once, so the next fill can be granted.
84 await this.#arm(answer.reason === 'new' ? Date.now() : Date.now() + WATCH_TICK_MS);
85 return answer;
86 }
88 async seen(repo: string, refs: [string, string][]): Promise<boolean> {
89 const state = await this.#load();
90 const differs = sawRefs(state, repo, refs);
91 if (differs) {
92 await this.#save();
93 await this.#arm(Date.now());
94 console.log(JSON.stringify({ msg: 'ply.refswatch', event: 'seen_mismatch', repo }));
95 }
96 return differs;
97 }
99 /** The mirror pushed: poll now. */
100 async poke(repos: string[]): Promise<void> {
101 const state = await this.#load();
102 if (repos.some(r => recordGet(state.repos, r).tag === 'some')) await this.#arm(Date.now());
103 }
105 /** The minute cron: an alarm that should be set but is not is set for now (that tick then sees the gap and purges). */
106 async ensure(): Promise<{ repos: number; alarm: number | null; polls: number }> {
107 const state = await this.#load();
108 if (needsAlarm(state) && (await this.ctx.storage.getAlarm()) === null) await this.ctx.storage.setAlarm(Date.now());
109 const polls = state.polls || 0;
110 state.polls = 0;
111 await this.#save();
112 return { repos: Object.keys(state.repos).length, alarm: await this.ctx.storage.getAlarm(), polls };
113 }
115 async status(): Promise<WatchState> {
116 return this.#load();
117 }
119 override async alarm(): Promise<void> {
120 if (this.#ticking) return;
121 this.#ticking = true;
122 try {
123 await this.#tick();
124 } finally {
125 this.#ticking = false;
126 }
127 }
129 async #tick(): Promise<void> {
130 const t0 = Date.now();
131 const state = await this.#load();
132 const tokens = this.#tokenSource();
133 const repos = pollList(state);
134 const polls: Record<string, PollResult> = {};
135 await Promise.all(repos.map(async repo => {
136 polls[repo] = tokens === null ? { ok: false, error: 'no PUBLIC_REPOS binding' } : await pollRefs(repo, tokens);
137 }));
138 state.polls = (state.polls || 0) + repos.length;
139 const pollMs = Date.now() - t0;
140 const now = Date.now();
141 const report = applyTick(state, polls, now);
142 let purge: PurgeAnswer | null = null;
143 if (report.purge.length > 0) {
144 purge = await this.#purge(report.purge);
145 if (!purge.ok) purgeFailed(state, report.purge);
146 }
147 await this.#save();
148 if (needsAlarm(state)) await this.ctx.storage.setAlarm(t0 + WATCH_TICK_MS);
149 const scrub = scrubSecrets(tokens === null ? [] : tokens.secrets());
150 if (report.purge.length > 0 || report.failed.length > 0 || report.gap || report.expired.length > 0) {
151 console.log(scrub(JSON.stringify({
152 msg: 'ply.refswatch', event: 'tick', repos: repos.length, poll_ms: pollMs, since_ms: report.sinceMs, gap: report.gap,
153 changed: report.changed, again: report.again, failed: report.failed.map(r => `${r}: ${pollError(polls, r)}`),
154 expired: report.expired, purge: report.purge, purge_ok: purge === null ? null : purge.ok, purge_ms: purge === null ? null : purge.ms,
155 purge_error: purge === null ? '' : purge.error,
156 })));
157 }
158 }
160 async #purge(repos: string[]): Promise<PurgeAnswer> {
161 const found = dig(this.ctx, ['exports', 'GitCache']);
162 if (found.tag === 'none' || !found.value) return { ok: false, ms: 0, error: 'no GitCache entrypoint' };
163 try {
164 return await (found.value as GitCacheRpc).purgeRefs(repos);
165 } catch (e) {
166 return { ok: false, ms: 0, error: errorText(e) };
167 }
168 }
Diagnostics4 git calls · 26 ms
CallStatusSizeTime026 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2002.6 KiB · hit21 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=21
git.response.bytes=2616
git.response.cache=HIT
git.response.duration_ms=21
git.fetch.wants=ac5ad607fead15a40414b516aadc322906f9d61c
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2616
git.pkt.payload.max=2595
git.sideband.band1.bytes=2594
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e308d359fa15db
Request
7632fe6f…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format