main
1// Mirrors in the public namespace follow their GitHub branch: a Worker cron (cloudflare.config.ts, every minute) compares
2// the source ref with the mirror's ref and, when they differ, fetches only the new objects from the source
3// (`want <source tip>`, `have <mirror tip>`) and pushes them to the mirror as a CAS update from the mirror's tip.
4//
5// Credentials: the source is read anonymously. The mirror is read with the binding's read token and written with a
6// write token minted through the same namespace-scoped binding for that one repo, with the minimum TTL (60 s), and
7// revoked as soon as the push returns. Nothing long-lived, nothing in CI. The token is never logged: the log line
8// carries oids, counts and timings only.
9//
10// Scope: a fixed list (MIRRORS). A source that rewrote history is followed too (Artifacts accepts a non-fast-forward
11// update with the right old oid). An update larger than MAX_SYNC_PACK is refused and logged; publish it with
12// scripts/publish.mjs, which chunks.
13import { chainGet, fromNullable, mapGet } from './absent.ts';
14import { PUBLIC_GIT_ORIGIN, publicRemote, type ArtifactsNamespace, type FetchLike, type TokenSource } from './public.ts';
15import { checkRemote, createGuardedFetch, redirectOptions } from './remote.ts';
16import { Tracer, ZERO_OID, arrayExporter, createTransport, fetchPack, lsRefs, push } from 'ply';
17import { MIRROR_CAPS, MIRROR_LIMITS } from './limits.ts';
18import type { Transport } from 'ply/transport';
19import { refname, unwrap, unwrapAsync } from './ply-result.ts';
21export type Mirror = { repo: string; source: string; branch: string };
23export const MIRRORS: readonly Mirror[] = Object.freeze([
24 { repo: 'ply', source: 'https://github.com/ivarvong/ply.git', branch: 'main' },
25 { repo: 'ply-web', source: 'https://github.com/ivarvong/ply-web.git', branch: 'main' },
26]);
28export const MAX_SYNC_PACK = MIRROR_CAPS.maxPackBytes;
29const LIMITS = MIRROR_LIMITS;
30export const WRITE_TTL_S = 60;
32export type SyncResult = {
33 repo: string;
34 outcome: 'in-sync' | 'updated' | 'error';
35 from: string | null;
36 to: string | null;
37 objects: number | null;
38 pushBytes: number | null;
39 ms: number;
40 code: string | null;
41 message: string | null;
42};
44export type SyncDeps = {
45 binding: ArtifactsNamespace;
46 tokens: TokenSource;
47 fetchImpl: FetchLike | null;
48 origin: string | null;
49};
51function tracer() {
52 return new Tracer({ exporter: arrayExporter(), resource: { 'service.name': 'ply-mirror' } });
53}
55const defaultFetch: FetchLike = (input, init) => {
56 const given = fromNullable(init);
57 if (given.tag === 'none') return fetch(input);
58 return fetch(input, given.value);
59};
61function releaseStub(handle: unknown): void {
62 const dispose = (handle as Partial<Disposable>)[Symbol.dispose];
63 const present = fromNullable(dispose);
64 if (present.tag === 'none') return;
65 const fn = present.value as (this: unknown) => void;
66 fn.call(handle);
67}
69async function tip(t: Transport, branch: string): Promise<string | null> {
70 const ref = `refs/heads/${branch}`;
71 const { refs } = await unwrapAsync(lsRefs(t, [ref]));
72 const row = mapGet(refs, ref);
73 if (row.tag === 'none') return null;
74 return row.value.oid;
75}
77function resolvedOrigin(deps: SyncDeps): string {
78 const origin = chainGet(deps, 'origin');
79 return origin.tag === 'some' && typeof origin.value === 'string' ? origin.value : PUBLIC_GIT_ORIGIN;
80}
82function allowOrigin(deps: SyncDeps): string[] {
83 const origin = chainGet(deps, 'origin');
84 if (origin.tag === 'none' || typeof origin.value !== 'string' || origin.value === '') return [];
85 return [origin.value];
86}
88function resolvedFetch(deps: SyncDeps): FetchLike {
89 const fetchImpl = chainGet(deps, 'fetchImpl');
90 if (fetchImpl.tag === 'some' && typeof fetchImpl.value === 'function') return fetchImpl.value as FetchLike;
91 return defaultFetch;
92}
94export async function syncMirror(m: Mirror, deps: SyncDeps): Promise<SyncResult> {
95 const t0 = Date.now();
96 const raw = resolvedFetch(deps);
97 const origin = resolvedOrigin(deps);
98 const allow = allowOrigin(deps);
99 const mirrorUrl = publicRemote(m.repo, origin);
100 const blank = { from: null, to: null, objects: null, pushBytes: null, code: null, message: null };
101 try {
102 const src = checkRemote(m.source);
103 const srcT = unwrap(createTransport(src.href, { tracer: tracer(), limits: LIMITS, quirks: {}, fetchImpl: createGuardedFetch(raw, src), ...redirectOptions(src) }));
104 const dst = checkRemote(mirrorUrl, allow);
105 const read = await deps.tokens.readToken(m.repo);
106 const dstT = unwrap(createTransport(dst.href, { tracer: tracer(), limits: LIMITS, quirks: {}, auth: { bearer: read }, fetchImpl: createGuardedFetch(raw, dst, allow), ...redirectOptions(dst, allow) }));
107 const [want, have] = await Promise.all([tip(srcT, m.branch), tip(dstT, m.branch)]);
108 if (!want) return { repo: m.repo, outcome: 'error', ...blank, code: 'not_found', message: `source has no ${m.branch}`, ms: Date.now() - t0 };
109 if (want === have) return { repo: m.repo, outcome: 'in-sync', ...blank, from: have, to: want, ms: Date.now() - t0 };
110 const r = await unwrapAsync(fetchPack(srcT, { wants: [want], haves: have ? [have] : [], ofsDelta: true, thinPack: false, done: true }, { resolve: true }));
111 // Resolved objects are whole and push as they are: their source-pack base is `packBase`, which push does not read
112 // (ply#217, fixed in ply 40083ea; until then this mapped each object to { type, data }).
113 const objects = r.objects ? [...r.objects.values()] : [];
114 const handle = await deps.binding.get(m.repo);
115 let writeToken = '';
116 try {
117 const minted = await (handle as unknown as { createToken(scope: 'write', ttl: number): Promise<{ plaintext: string }> }).createToken('write', WRITE_TTL_S);
118 const plain = chainGet(minted, 'plaintext');
119 writeToken = plain.tag === 'some' ? String(plain.value) : '';
120 if (!writeToken) throw Object.assign(new Error('no write token'), { code: 'upstream_5xx' });
121 const dstW = unwrap(createTransport(dst.href, { tracer: tracer(), limits: LIMITS, quirks: {}, auth: { bearer: writeToken }, fetchImpl: createGuardedFetch(raw, dst, allow), ...redirectOptions(dst, allow) }));
122 const pushed = await unwrapAsync(push(dstW, { ref: refname(`refs/heads/${m.branch}`), oldOid: have === null ? ZERO_OID : have, newOid: want, objects }));
123 const bytes = chainGet(pushed, 'packBytes');
124 const pushBytes = bytes.tag === 'some' && typeof bytes.value === 'number' ? bytes.value : 0;
125 return { repo: m.repo, outcome: 'updated', ...blank, from: have, to: want, objects: objects.length, pushBytes, ms: Date.now() - t0 };
126 } finally {
127 const revoke = chainGet(handle, 'revokeToken');
128 if (writeToken && revoke.tag === 'some' && typeof revoke.value === 'function') {
129 try { await (revoke.value as (token: string) => Promise<unknown>).call(handle, writeToken); } catch { /* it expires in WRITE_TTL_S */ }
130 }
131 try { releaseStub(handle); } catch { /* best effort */ }
132 }
133 } catch (e) {
134 const codeField = chainGet(e, 'code');
135 const messageField = chainGet(e, 'message');
136 const code = codeField.tag === 'some' && typeof codeField.value === 'string' ? codeField.value : 'internal';
137 const message = String(messageField.tag === 'some' ? messageField.value : e).replace(/art_v1_[0-9a-f]+(\?expires=\d+)?/g, '[redacted]').slice(0, 300);
138 return { repo: m.repo, outcome: 'error', ...blank, code, message, ms: Date.now() - t0 };
139 }
140}
142export async function syncAll(deps: SyncDeps, mirrors: readonly Mirror[] = MIRRORS): Promise<SyncResult[]> {
143 return Promise.all(mirrors.map(m => syncMirror(m, deps)));
144}
Diagnostics4 git calls · 28 ms
CallStatusSizeTime028 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2002.8 KiB · hit21 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=21 git.response.bytes=2831 git.response.cache=HIT git.response.duration_ms=21 git.fetch.wants=98633062e55460c7b76bd520c6fed55abb07b760 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2831 git.pkt.payload.max=2810 git.sideband.band1.bytes=2809 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2ee571d1b77be
- Request
- b60b77db…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format