GitHubivarvong
ply
main
1// Resumable transfer, the pack a step reads (#260): a resumed request rebuilds its TransferPack from a saved index
2// (packsaved.ts) when the caller kept one, and scans and resolves the spool only when it has none or the one it has is
3// refused. The pack is named by the spool's own last 20 bytes (its trailer) and length, so an index saved for another
4// spool, or a spool replaced under the same name, is never used.
5//
6// loadTransferPack(dst, source, saved, opts) → { from: TransferPack, loaded, saved }
7// loaded: 'saved' the saved index and resolved columns, no read of the spool but its trailer
8// 'resolved' the saved index (it had no resolved columns), resolved again over the spool
9// 'scanned' scanned and resolved; `refused` says why a saved index was not used (none: there was none)
10// saved: the bytes to keep for the next request (the saved ones when they were used, else a fresh encoding)
11//
12// Limits: opts.resolve.limits is parsed once, before anything is read (one that does not parse is bad_request), and
13// is what both the resolve and the saved index's chain check run under: an index saved under a raised maxDeltaChain is
14// used under that limit again, and refused (then rescanned, and the resolve says limit_exceeded) under a lower one.
15// The saved index's entry count is held to opts.scan.limits.maxPackObjects, the scan's cap (parsed here too); over it
16// the index is refused and rescanned like any other refusal.
17//
18// Span: git.transfer.load (under dst's parent) with git.transfer.load.from, .refused, .saved_bytes, .entries and
19// git.pack.trailer; a scan's refusal is the span's error.
20import { argsError } from './errors.js';
21import type { PlyError } from './errors.js';
22import type { Option, Result } from './result.js';
23import { ok, err, some, none } from './result.js';
24import type { Input } from './absent.js';
25import { withDefaults } from './absent.js';
26import type { ScanIndexOpts, ScannedPack } from './packscanindex.js';
27import { scanPackIndex, scanLimitsOf } from './packscanindex.js';
28import type { ResolveRangeOpts } from './packrangeresolve.js';
29import { resolvePackRange, resolveOptsOf } from './packrangeresolve.js';
30import type { RangeSource } from './rangesource.js';
31import { UNCANCELLABLE, HALTED, haltOn, cancelLate, aborted } from './rangesource.js';
32import type { TransferPack } from './transferplan.js';
33import type { SavedIndex, SavedIndexRefusal } from './packsaved.js';
34import { decodeSavedBytes, encodeSavedIndex, describeRefusal, packIdentity } from './packsaved.js';
35import type { Span } from './otel.js';
36import { KIND } from './otel.js';
37import type { Transport } from './transport.js';
38import { spanned } from './transport.js';
40export type Loaded = { readonly tag: 'saved' } | { readonly tag: 'resolved' } | { readonly tag: 'scanned'; readonly refused: Option<SavedIndexRefusal> };
41export type LoadedPack = { readonly from: TransferPack; readonly loaded: Loaded; readonly saved: Uint8Array };
42// `signal` aborts the load wherever it is (the trailer read, the scan, the resolve): each await on `source` is raced
43// against it, so the call settles `aborted` at once even when the spool never answers. It is the signal of the scan and
44// the resolve too (one in `scan` or `resolve` is replaced). Default: never aborts.
45export type LoadConfig = { readonly scan: Partial<ScanIndexOpts>; readonly resolve: Partial<ResolveRangeOpts>; readonly signal: AbortSignal };
46export type LoadOpts = Input<LoadConfig>;
48async function resolved(source: RangeSource, pack: ScannedPack, cfg: LoadConfig, loaded: Loaded): Promise<Result<LoadedPack, PlyError>> {
49 const r = await resolvePackRange(pack, source, { ...cfg.resolve, signal: cfg.signal });
50 if (!r.ok) return r;
51 const saved = encodeSavedIndex(pack, some(r.value));
52 return saved.ok ? ok({ from: { source, pack, resolved: r.value }, loaded, saved: saved.value }) : saved;
53}
54async function scanned(source: RangeSource, cfg: LoadConfig, refused: Option<SavedIndexRefusal>): Promise<Result<LoadedPack, PlyError>> {
55 // opening the spool is raced here; reading it is the scan's, raced there against the same signal
56 const halt = haltOn(cfg.signal);
57 const whole = await halt.race(source.read(0, source.size, halt.signal), cancelLate).finally(halt.release);
58 if (whole === HALTED) return err(aborted('transfer load'));
59 if (!whole.ok) return whole;
60 const pack = await scanPackIndex(whole.value, { ...cfg.scan, signal: cfg.signal });
61 if (!pack.ok) return pack;
62 return resolved(source, pack.value, cfg, { tag: 'scanned', refused });
63}
65// A saved index read back: used as it is, resolved again when it has no resolved columns, or refused (a scan, and why).
66async function fromSaved(source: RangeSource, cfg: LoadConfig, span: Span, bytes: Uint8Array, decoded: Result<SavedIndex, SavedIndexRefusal>): Promise<Result<LoadedPack, PlyError>> {
67 if (!decoded.ok) {
68 // every refusal, limit_exceeded too, is rescanned: the stored count is unverified bytes (checked before the SHA-1
69 // and the pack identity), and a pack really over the cap is refused by the scan's own header-count check
70 span.set({ 'git.transfer.load.refused': decoded.error.tag, 'git.transfer.load.refused_reason': describeRefusal(decoded.error) });
71 const again = await scanned(source, cfg, some(decoded.error));
72 return again;
73 }
74 const { pack, resolved: kept } = decoded.value;
75 if (kept.tag === 'none') {
76 const r = await resolved(source, pack, cfg, { tag: 'resolved' });
77 return r;
78 }
79 return ok({ from: { source, pack, resolved: kept.value }, loaded: { tag: 'saved' }, saved: bytes });
80}
82// See the top of this file.
83export async function loadTransferPack(dst: Transport, source: RangeSource, saved: Option<Uint8Array>, opts: LoadOpts = {}): Promise<Result<LoadedPack, PlyError>> {
84 const shape = argsError('loadTransferPack', [['dst', dst, 'object'], ['source', source, 'object'], ['saved', saved, 'object'], ['options', opts, 'object']]);
85 if (shape !== null) return shape;
86 const input = withDefaults<LoadConfig>(opts, { scan: {}, resolve: {}, signal: UNCANCELLABLE });
87 const span0 = { parent: dst.parent, kind: KIND.INTERNAL, attrs: { 'git.transfer.load.saved_in': saved.tag === 'some' ? saved.value.length : 0, 'git.transfer.spool.bytes': source.size }, scrub: dst.scrub };
88 const r = await spanned(dst, 'git.transfer.load', span0, async (span: Span): Promise<Result<LoadedPack, PlyError>> => {
89 // the caller's resolve options, read once as the resolver reads them (an inherited one counts), limits parsed:
90 // the saved index is held to the chain limit its resolve would run under
91 const resolve = resolveOptsOf(input.resolve);
92 if (!resolve.ok) return resolve;
93 // the entry-count bound is the scan's cap (scan.limits.maxPackObjects), parsed as the scan parses it
94 const scanLimits = scanLimitsOf(input.scan);
95 if (!scanLimits.ok) return scanLimits;
96 const cfg: LoadConfig = { scan: input.scan, resolve: resolve.value, signal: input.signal };
97 const id = await packIdentity(source, cfg.signal);
98 if (!id.ok) return id;
99 span.set({ 'git.pack.trailer': id.value.trailer });
100 const out = saved.tag === 'none' ? await scanned(source, cfg, none) : await fromSaved(source, cfg, span, saved.value, decodeSavedBytes(saved.value, id.value, resolve.value.limits.maxDeltaChain, scanLimits.value.maxPackObjects));
101 if (out.ok) span.set({ 'git.transfer.load.from': out.value.loaded.tag, 'git.transfer.load.saved_bytes': out.value.saved.length, 'git.transfer.load.entries': out.value.from.pack.count });
102 return out;
103 });
104 return r;
105}
Diagnostics6 git calls · 147 ms
CallStatusSizeTime0147 ms
discover200191 B4 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB · hit4 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=11990 git.response.cache=HIT git.response.duration_ms=4 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02002.4 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=136 git.response.bytes=2507 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2507 git.pkt.payload.max=2480 git.sideband.band1.bytes=2480 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=2480 git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381Open pack map
fetch2002.6 KiB136 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=2660 git.response.cache=MISS git.response.duration_ms=136 git.fetch.wants=e908cec9fbb565226ae05b6ae1541651229c484d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2660 git.pkt.payload.max=2633 git.sideband.band1.bytes=2633 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e0fdb29d93332c
- Request
- 9b9ee67a…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format