// Resumable transfer, the pack a step reads (#260): a resumed request rebuilds its TransferPack from a saved index // (packsaved.ts) when the caller kept one, and scans and resolves the spool only when it has none or the one it has is // refused. The pack is named by the spool's own last 20 bytes (its trailer) and length, so an index saved for another // spool, or a spool replaced under the same name, is never used. // // loadTransferPack(dst, source, saved, opts) → { from: TransferPack, loaded, saved } // loaded: 'saved' the saved index and resolved columns, no read of the spool but its trailer // 'resolved' the saved index (it had no resolved columns), resolved again over the spool // 'scanned' scanned and resolved; `refused` says why a saved index was not used (none: there was none) // saved: the bytes to keep for the next request (the saved ones when they were used, else a fresh encoding) // // Limits: opts.resolve.limits is parsed once, before anything is read (one that does not parse is bad_request), and // is what both the resolve and the saved index's chain check run under: an index saved under a raised maxDeltaChain is // used under that limit again, and refused (then rescanned, and the resolve says limit_exceeded) under a lower one. // The saved index's entry count is held to opts.scan.limits.maxPackObjects, the scan's cap (parsed here too); over it // the index is refused and rescanned like any other refusal. // // Span: git.transfer.load (under dst's parent) with git.transfer.load.from, .refused, .saved_bytes, .entries and // git.pack.trailer; a scan's refusal is the span's error. import { argsError } from './errors.js'; import type { PlyError } from './errors.js'; import type { Option, Result } from './result.js'; import { ok, err, some, none } from './result.js'; import type { Input } from './absent.js'; import { withDefaults } from './absent.js'; import type { ScanIndexOpts, ScannedPack } from './packscanindex.js'; import { scanPackIndex, scanLimitsOf } from './packscanindex.js'; import type { ResolveRangeOpts } from './packrangeresolve.js'; import { resolvePackRange, resolveOptsOf } from './packrangeresolve.js'; import type { RangeSource } from './rangesource.js'; import { UNCANCELLABLE, HALTED, haltOn, cancelLate, aborted } from './rangesource.js'; import type { TransferPack } from './transferplan.js'; import type { SavedIndex, SavedIndexRefusal } from './packsaved.js'; import { decodeSavedBytes, encodeSavedIndex, describeRefusal, packIdentity } from './packsaved.js'; import type { Span } from './otel.js'; import { KIND } from './otel.js'; import type { Transport } from './transport.js'; import { spanned } from './transport.js'; export type Loaded = { readonly tag: 'saved' } | { readonly tag: 'resolved' } | { readonly tag: 'scanned'; readonly refused: Option }; export type LoadedPack = { readonly from: TransferPack; readonly loaded: Loaded; readonly saved: Uint8Array }; // `signal` aborts the load wherever it is (the trailer read, the scan, the resolve): each await on `source` is raced // against it, so the call settles `aborted` at once even when the spool never answers. It is the signal of the scan and // the resolve too (one in `scan` or `resolve` is replaced). Default: never aborts. export type LoadConfig = { readonly scan: Partial; readonly resolve: Partial; readonly signal: AbortSignal }; export type LoadOpts = Input; async function resolved(source: RangeSource, pack: ScannedPack, cfg: LoadConfig, loaded: Loaded): Promise> { const r = await resolvePackRange(pack, source, { ...cfg.resolve, signal: cfg.signal }); if (!r.ok) return r; const saved = encodeSavedIndex(pack, some(r.value)); return saved.ok ? ok({ from: { source, pack, resolved: r.value }, loaded, saved: saved.value }) : saved; } async function scanned(source: RangeSource, cfg: LoadConfig, refused: Option): Promise> { // opening the spool is raced here; reading it is the scan's, raced there against the same signal const halt = haltOn(cfg.signal); const whole = await halt.race(source.read(0, source.size, halt.signal), cancelLate).finally(halt.release); if (whole === HALTED) return err(aborted('transfer load')); if (!whole.ok) return whole; const pack = await scanPackIndex(whole.value, { ...cfg.scan, signal: cfg.signal }); if (!pack.ok) return pack; return resolved(source, pack.value, cfg, { tag: 'scanned', refused }); } // A saved index read back: used as it is, resolved again when it has no resolved columns, or refused (a scan, and why). async function fromSaved(source: RangeSource, cfg: LoadConfig, span: Span, bytes: Uint8Array, decoded: Result): Promise> { if (!decoded.ok) { // every refusal, limit_exceeded too, is rescanned: the stored count is unverified bytes (checked before the SHA-1 // and the pack identity), and a pack really over the cap is refused by the scan's own header-count check span.set({ 'git.transfer.load.refused': decoded.error.tag, 'git.transfer.load.refused_reason': describeRefusal(decoded.error) }); const again = await scanned(source, cfg, some(decoded.error)); return again; } const { pack, resolved: kept } = decoded.value; if (kept.tag === 'none') { const r = await resolved(source, pack, cfg, { tag: 'resolved' }); return r; } return ok({ from: { source, pack, resolved: kept.value }, loaded: { tag: 'saved' }, saved: bytes }); } // See the top of this file. export async function loadTransferPack(dst: Transport, source: RangeSource, saved: Option, opts: LoadOpts = {}): Promise> { const shape = argsError('loadTransferPack', [['dst', dst, 'object'], ['source', source, 'object'], ['saved', saved, 'object'], ['options', opts, 'object']]); if (shape !== null) return shape; const input = withDefaults(opts, { scan: {}, resolve: {}, signal: UNCANCELLABLE }); const span0 = { parent: dst.parent, kind: KIND.INTERNAL, attrs: { 'git.transfer.load.saved_in': saved.tag === 'some' ? saved.value.length : 0, 'git.transfer.spool.bytes': source.size }, scrub: dst.scrub }; const r = await spanned(dst, 'git.transfer.load', span0, async (span: Span): Promise> => { // the caller's resolve options, read once as the resolver reads them (an inherited one counts), limits parsed: // the saved index is held to the chain limit its resolve would run under const resolve = resolveOptsOf(input.resolve); if (!resolve.ok) return resolve; // the entry-count bound is the scan's cap (scan.limits.maxPackObjects), parsed as the scan parses it const scanLimits = scanLimitsOf(input.scan); if (!scanLimits.ok) return scanLimits; const cfg: LoadConfig = { scan: input.scan, resolve: resolve.value, signal: input.signal }; const id = await packIdentity(source, cfg.signal); if (!id.ok) return id; span.set({ 'git.pack.trailer': id.value.trailer }); const out = saved.tag === 'none' ? await scanned(source, cfg, none) : await fromSaved(source, cfg, span, saved.value, decodeSavedBytes(saved.value, id.value, resolve.value.limits.maxDeltaChain, scanLimits.value.maxPackObjects)); if (out.ok) span.set({ 'git.transfer.load.from': out.value.loaded.tag, 'git.transfer.load.saved_bytes': out.value.saved.length, 'git.transfer.load.entries': out.value.from.pack.count }); return out; }); return r; }