GitHubivarvong

ply

main
ply6 git calls83 ms25 KiB4 cachedprotocol v2
206 lines14 KiBRaw
1// Mirror one ref from a source remote to a destination: `git fetch <src> <ref>` then
2// `git push --force-with-lease=<dstRef>:<dstTip> <dst> <tip>:<dstRef>`, with no local repository.
3//
4// syncRef(srcUrl, dstUrl, { ref, dstRef?, force? }, { src?, dst?, tracer?, limits?, signal?, verify? })
5// → { outcome: 'in_sync', ref, dstRef, oid }
6// | { outcome: 'created' | 'updated', ref, dstRef, from, to, forced, objects, pushBytes }
7//
8// One ls-refs on each side (concurrently). Equal tips: `in_sync`, nothing else is sent. Otherwise one fetch from the
9// source wanting its tip with the destination's tip as the one `have` (a have the source does not know is harmless: it
10// NAKs and sends the whole history), then one receive-pack command on the destination whose old id is the tip ls-refs
11// read there (the zero id when the ref is absent: a create). The pack sent is the source's pack as received, object for
12// object and delta for delta (an OFS_DELTA stays one against the same base; nothing is re-deltified or made thin), so it
13// holds only what the source computed the destination lacks: nothing reachable from the destination's tip when the
14// source has that commit. The fetch never asks for a thin pack: every base is in the pack, so the destination needs no
15// object it was not shown to have.
16//
17// Fast-forward: the destination's tip is a parent of a commit in the received pack reachable from the source's tip
18// (the pack holds exactly the commits the destination's tip does not reach, so the path from the source's tip down to
19// an ancestor tip is in it). An update that is not a fast-forward (a rewritten or diverged source, or a destination
20// ahead of the source), and any change to an existing tag (refs/tags/), is `not_fast_forward` unless `force` (as
21// pushRepo refuses one; the default is false, as `git push` without `+`). Forced, it is pushed with the same CAS.
22//
23// A push that loses the compare-and-swap (the destination moved after ls-refs) re-reads the destination ref:
24// `cas_conflict` with `expected` the tip ply saw (zero for a create) and `actual` the tip now (null: deleted), as
25// refops' createBranch and deleteBranch report a lost race. Anything else the destination refuses is its own error.
26//
27// Options. Each remote has its own credential and transport: `src` and `dst` each take `auth`, `fetchImpl`, `quirks`
28// and `transport` (a mirror reads with one token and writes with another, often on two hosts). One operation has one
29// root span, one abort signal and one budget, so `tracer`, `parent`, `signal` and `limits` are shared; a top-level
30// `fetchImpl` is the default for both sides. A top-level `auth` names neither side and is refused (`bad_request`,
31// `what: 'auth'`) rather than guessed. `limits.maxPackBytes` bounds both the pack fetched and the pack pushed.
32/* oxlint-disable require-await -- the entry is async: a refused argument is an Err in a promise, like every other answer (read.ts) */
33/* oxlint-disable import/max-dependencies -- an operation over ls-refs on two remotes, the fetch, the pack and the push */
34import type { PlyError } from './errors.js';
35import type { Result } from './result.js';
36import type { Input } from './absent.js';
37import type { EntryConfig } from './boundary.js';
38import type { OperationOpts } from './operation.js';
39import type { OpContext } from './merge.js';
40import type { Transport, TransportInput, Scrub } from './transport.js';
41import type { PackObject } from './fetch.js';
42import type { PackInput } from './pack.js';
43import type { ParsedPack } from './packresolve.js';
44import type { CheckedLimits, LimitsInput } from './limits.js';
45import { fail } from './errors.js';
46import { ok, map, assertNever } from './result.js';
47import { field, mapGet, withDefaults } from './absent.js';
48import { lsRefs } from './lsrefs.js';
49import { fetchPack } from './fetchpack.js';
50import { push, ZERO_OID } from './push.js';
51import { operation, verifyRef } from './operation.js';
52import { createTransport, redactUrl } from './http.js';
53import { transportFrom, withParent } from './transport.js';
54import { parseCommit, parseTag, TYPE } from './objects.js';
55import { validatePushRef } from './validate.js';
56import { refOid } from './merge.js';
57import { DEFAULT_LIMITS, parseLimits } from './limits.js';
58import { Tracer } from './otel.js';
59import type { Span } from './otel.js';
61// One remote's connection: its credential, fetch, learned quirks, or a transport the caller already holds.
62export type SyncSideConfig = Pick<EntryConfig, 'auth' | 'quirks' | 'transport'> & { readonly fetchImpl: EntryConfig['fetchImpl'] | null };
63export type SyncSide = Input<SyncSideConfig>;
64type Shared = { readonly tracer: Tracer; readonly parent: Span | null; readonly limits: LimitsInput; readonly signal: AbortSignal | null; readonly fetchImpl: EntryConfig['fetchImpl'] | null; readonly verify: boolean; readonly src: SyncSide; readonly dst: SyncSide };
65export type SyncRefOptions = Input<Shared>;
66export type SyncRefConfig = { readonly dstRef: string | null; readonly force: boolean };
67export type SyncRefSpec = { readonly ref: string } & Input<SyncRefConfig>;
68export type SyncRefResult =
69 | { readonly outcome: 'in_sync'; readonly ref: string; readonly dstRef: string; readonly oid: string }
70 | { readonly outcome: 'created' | 'updated'; readonly ref: string; readonly dstRef: string; readonly from: string | null; readonly to: string; readonly forced: boolean; readonly objects: number; readonly pushBytes: number };
72const sideDefaults = (): SyncSideConfig => ({ auth: null, fetchImpl: null, quirks: null, transport: null });
74// The destination's transport, under the operation's root: the caller's (re-parented), or one built for the url.
75function dstTransport(url: string, side: SyncSideConfig, shared: Shared, root: Span): Result<Transport, PlyError> {
76 if (side.transport !== null) return map(transportFrom(side.transport), d => withParent(d, root));
77 const fetchImpl = side.fetchImpl === null ? shared.fetchImpl : side.fetchImpl;
78 return createTransport(url, { auth: side.auth, tracer: shared.tracer, parent: root, fetchImpl: fetchImpl === null ? globalThis.fetch : fetchImpl, limits: shared.limits, signal: shared.signal, ...(side.quirks === null ? {} : { quirks: side.quirks }) });
81// Both credentials redacted from the root span and from the error the caller gets.
82const bothScrubs = (a: Scrub | null, b: Scrub | null): Scrub | null => {
83 if (a === null) return b;
84 if (b === null) return a;
85 return (s: string): string => b(a(s));
86};
88// Is `old` reachable from `tip` through the received pack? `tip` is peeled (an annotated tag to its object); the walk
89// follows commit parents within the pack and stops at a parent outside it (one the destination's tip already reaches).
90function reaches(objects: ReadonlyMap<string, PackObject>, tip: string, old: string, limits: CheckedLimits): Result<boolean, PlyError> {
91 const stack = [tip];
92 const seen = new Set<string>();
93 for (let oid = stack.pop(); typeof oid === 'string'; oid = stack.pop()) {
94 if (oid === old) return ok(true);
95 const o = mapGet(objects, oid);
96 if (seen.has(oid) || o.tag === 'none') continue;
97 seen.add(oid);
98 if (o.value.type === TYPE.TAG) {
99 const tag = parseTag(o.value.data); if (!tag.ok) return tag;
100 stack.push(tag.value.object);
101 } else if (o.value.type === TYPE.COMMIT) {
102 const c = parseCommit(o.value.data, limits); if (!c.ok) return c;
103 stack.push(...c.value.parents);
104 }
105 }
106 return ok(false);
109// The received pack, as push sends it: each object in the order it arrived, a delta against the same base (an OFS base
110// as the index of that earlier object, a REF base by id), a whole object whole.
111function forwarded(parsed: ParsedPack | null): PackInput[] {
112 if (parsed === null) return [];
113 const index = new Map(parsed.objects.map((o, i) => [o.offset, i]));
114 return parsed.objects.map((o): PackInput => {
115 switch (o.base.tag) {
116 case 'ofs': {
117 const at = mapGet(index, o.base.offset);
118 return at.tag === 'some' ? { type: o.type, data: o.data, base: { tag: 'ofs', index: at.value } } : { type: o.type, data: o.data };
119 }
120 case 'ref': return { type: o.type, data: o.data, base: o.base };
121 case 'none': return { type: o.type, data: o.data };
122 }
123 // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check
124 return assertNever(o.base);
125 });
128// Effect: a push the destination refused as stale (it moved after ls-refs). receive-pack does not say where it went, so
129// re-read it: moved, cas_conflict naming the tip it has now (null: deleted); unchanged, the push's own error stands.
130async function lostRace(d: Transport, dstRef: string, expected: string, pushed: Result<never, PlyError>): Promise<Result<never, PlyError>> {
131 const now = await lsRefs(d, [dstRef]);
132 if (!now.ok) return now;
133 const tip = refOid(now.value.refs, dstRef);
134 const actual = tip.tag === 'some' ? tip.value : null;
135 if (actual === (expected === ZERO_OID ? null : expected)) return pushed;
136 return fail('cas_conflict', `${dstRef} moved from ${expected} to ${actual === null ? 'absent' : actual} during the sync`, { what: 'ref', expected, actual });
139type Parsed = { readonly ref: string; readonly dstRef: string; readonly force: boolean; readonly shared: Shared; readonly src: SyncSideConfig; readonly dst: SyncSideConfig };
140// The spec and options, total, or the caller's mistake (nothing is sent).
141function parseSync(spec: SyncRefSpec, options: SyncRefOptions): Result<Parsed, PlyError> {
142 const { ref } = spec;
143 const named = validatePushRef(ref, false); if (!named.ok) return named;
144 const { dstRef: givenDst, force } = withDefaults<SyncRefConfig>(spec, { dstRef: null, force: false });
145 const dstRef = givenDst === null ? ref : givenDst;
146 const dstNamed = validatePushRef(dstRef, false); if (!dstNamed.ok) return dstNamed;
147 if (typeof force !== 'boolean') return fail('bad_request', 'syncRef: force must be a boolean', { what: 'force' });
148 if (field(options, 'auth').tag === 'some') return fail('bad_request', 'syncRef: auth belongs to one remote: pass opts.src.auth and opts.dst.auth', { what: 'auth' });
149 const given = withDefaults<Shared>(options, { tracer: new Tracer(), parent: null, limits: DEFAULT_LIMITS, signal: null, fetchImpl: null, verify: false, src: {}, dst: {} });
150 // parsed once here, for both transports: a bad one is refused before either is made
151 const limits = parseLimits(given.limits); if (!limits.ok) return limits;
152 const shared = { ...given, limits: limits.value };
153 return ok({ ref, dstRef, force, shared, src: withDefaults(shared.src, sideDefaults()), dst: withDefaults(shared.dst, sideDefaults()) });
156// Effect: the fetch from the source (the destination's tip as the have), the fast-forward decision, and the CAS push.
157type Transfer = { readonly t: Transport; readonly d: Transport; readonly root: Span; readonly to: string; readonly from: string | null };
158async function transfer({ dstRef, force, shared }: Parsed, { t, d, root, to, from }: Transfer): Promise<Result<Omit<SyncRefResult & { readonly outcome: 'created' | 'updated' }, 'ref'>, PlyError>> {
159 const fetched = await fetchPack(t, { wants: [to], haves: from === null ? [] : [from], done: true, ofsDelta: true, thinPack: false }, { resolve: true });
160 if (!fetched.ok) return fetched;
161 const objects = fetched.value.objects === null ? new Map<string, PackObject>() : fetched.value.objects;
162 const tag = dstRef.startsWith('refs/tags/');
163 let fastForward = true;
164 if (from !== null) {
165 const ff = reaches(objects, to, from, t.limits); if (!ff.ok) return ff;
166 fastForward = ff.value && !tag;
167 if (!fastForward && !force) return fail('not_fast_forward', `${dstRef} is at ${from} on the destination and ${tag ? 'the tag already exists there' : `${to} does not descend from it`}; pass force: true to overwrite it`, { what: 'ref', got: [dstRef] });
168 }
169 root.set({ 'git.sync_ref.fast_forward': fastForward });
170 const pack = forwarded(fetched.value.parsed);
171 const expected = from === null ? ZERO_OID : from;
172 const pushed = await push(d, { ref: dstRef, oldOid: expected, newOid: to, objects: pack });
173 if (!pushed.ok) return pushed.error.code === 'cas_conflict' ? lostRace(d, dstRef, expected, pushed) : pushed;
174 if (shared.verify) { const seen = await verifyRef(d, root, 'git.sync_ref', dstRef, to); if (!seen.ok) return seen; }
175 return ok({ outcome: from === null ? 'created' : 'updated', dstRef, from, to, forced: !fastForward, objects: pack.length, pushBytes: pushed.value.packBytes });
178export async function syncRef(srcUrl: string, dstUrl: string, spec: SyncRefSpec, options: SyncRefOptions = {}): Promise<Result<SyncRefResult, PlyError>> {
179 const parsed = parseSync(spec, options); if (!parsed.ok) return parsed;
180 const p = parsed.value;
181 const { ref, dstRef, force, shared, src, dst } = p;
182 const srcTransport: TransportInput | null = src.transport;
183 const opOpts: OperationOpts = {
184 tracer: shared.tracer, parent: shared.parent, limits: shared.limits, signal: shared.signal, auth: src.auth, quirks: src.quirks, transport: srcTransport,
185 fetchImpl: src.fetchImpl === null ? shared.fetchImpl : src.fetchImpl,
186 attrs: { 'git.sync_ref.ref': ref, 'git.sync_ref.dst_ref': dstRef, 'git.sync_ref.dst_url': dst.transport === null ? redactUrl(dstUrl) : dst.transport.url, 'git.sync_ref.force': force },
187 };
188 return operation('sync_ref', srcUrl, opOpts, async ({ t, root, outcome }: OpContext): Promise<Result<SyncRefResult, PlyError>> => {
189 const made = dstTransport(dstUrl, dst, shared, root); if (!made.ok) return made;
190 const d = made.value;
191 root.scrub = bothScrubs(t.scrub, d.scrub);
192 const [srcListed, dstListed] = await Promise.all([lsRefs(t, [ref]), lsRefs(d, [dstRef])]);
193 if (!srcListed.ok) return srcListed;
194 if (!dstListed.ok) return dstListed;
195 const srcTip = refOid(srcListed.value.refs, ref);
196 if (srcTip.tag === 'none') return fail('not_found', `ref ${ref} not found on the source`, { miss: { kind: 'ref', ref, candidates: [ref] } });
197 const to = srcTip.value;
198 const dstTip = refOid(dstListed.value.refs, dstRef);
199 const from = dstTip.tag === 'some' ? dstTip.value : null;
200 root.set({ 'git.sync_ref.to': to, 'git.sync_ref.from': from === null ? '' : from });
201 if (from === to) return outcome('in_sync', { outcome: 'in_sync', ref, dstRef, oid: to });
202 const sent = await transfer(p, { t, d, root, to, from }); if (!sent.ok) return sent;
203 const result: SyncRefResult = { ...sent.value, ref };
204 return outcome(result.outcome, result, { 'git.sync_ref.objects': result.objects, 'git.sync_ref.push_bytes': result.pushBytes });
205 });
Diagnostics6 git calls · 83 ms
CallStatusSizeTime083 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02002.4 KiB · hit1 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=75
git.response.bytes=2507
git.response.cache=HIT
git.response.duration_ms=1
git.fetch.filter=tree:0
git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2507
git.pkt.payload.max=2480
git.sideband.band1.bytes=2480
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=2480
git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381
Open pack map
fetch2005.0 KiB75 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=5147
git.response.cache=MISS
git.response.duration_ms=75
git.fetch.wants=7d0861ecdbdd64f18015563e7afe878623925021
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5147
git.pkt.payload.max=5120
git.sideband.band1.bytes=5120
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0f83c4ef2332c
Request
bc395b10…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format