// v2 fetch: request body (pure), response state machine (pure step), the streamed band-1 pack, and fetchObject — one // object by oid through the single-object pack parser, hash-verified. The multi-object fetch (the pack read through // the scan as it arrives, then delta-resolved) lives in fetchpack.js. // The step functions, the checks and fetchObject return Result. packStream is a stream: it fails through the stream // protocol (its iterator rejects with the PlyError), and the exchanges that drain it catch that once (coded). /* oxlint-disable require-unicode-regexp -- ASCII wire text; `u` costs a byte per literal and changes what /i matches (U+017F, U+212A) */ import type { ErrorOf, Extra, PlyError } from './errors.js'; import type { CheckedLimits } from './limits.js'; import type { Span } from './otel.js'; import type { Input } from './absent.js'; import type { Option, Result } from './result.js'; import type { Attrs, Frame, Scrub, Transport } from './transport.js'; import { fail, scrubError, remoteQuoted, remoteText, REMOTE_TEXT_MAX } from './errors.js'; import { pkts, pktInit, pktStep, pktFinish, pktAttrs } from './pktline.js'; import { td, concat, take, decodeChunk } from './bytes.js'; import { Sha1, toHex } from './sha1.js'; import { TYPE, TYPE_NAME, hashObject, assertFetched, nonCommitType, parseTree } from './objects.js'; import type { ObjType, ObjectError, TreeEntry } from './objects.js'; // oxlint-disable-next-line import/max-dependencies -- the fetch path's parsers each live in their own module (pack, inflate, objects, sha1, cache) import { corruptCacheHit } from './cachestatus.js'; import { assertNever, err, getOrElse, mapOption, none, ok } from './result.js'; import type { InflateError, InflateStats, InflateStream } from './inflatestream.js'; import { inflateInit, inflateFeed, inflatePull, inflateEnd, inflateUnread, declaredBound, zlibWrap } from './inflatestream.js'; import { resolvePack, resolvePackCollect, packCaps, packCollectOpen, packCollectFeed, packCollectEnd, readObjHeader, readUint32BE } from './packresolve.js'; import type { PackCollector, ParsedPack, ScannedHeader } from './packresolve.js'; import type { HeaderError, ObjSize } from './packstream.js'; import { NO_SIZE_CAP } from './packstream.js'; import { atOr, fromNullish, mapGet, withDefaults } from './absent.js'; import { PACK_SIGNATURE, PACK_VERSION, PACK_HEADER_BYTES, PACK_TRAILER_BYTES, UINT32_BYTES, SIDEBAND_PACK, SIDEBAND_PROGRESS, SIDEBAND_ERROR, SIDEBAND_CHANNEL_BYTES, OID_HEX_CHARS, PROTOCOL_V2, zlibBound } from './format.js'; import { DEFAULT_LIMITS, limitExceeded } from './limits.js'; import { validateWire } from './validate.js'; import type { WireField } from './validate.js'; import { parseOid, type Oid } from './oid.js'; import { frames, advertise, coded, clientSpan, proto, harvestBytes, learnFilter, learnShallow, takesShallow, shallowLearned, probing, spanned } from './transport.js'; import { capLookup, parseCapabilitiesV2 } from './caps.js'; import { thrownCodeIs, thrownReasonIs, thrownCode, thrownMessage } from './boundary.js'; /* oxlint-disable no-magic-numbers -- span tails are diagnostics, not wire-format facts */ const PROGRESS_KEEP = 512; const PROGRESS_ATTR = 200; const OID_PREVIEW = 48; /* oxlint-enable no-magic-numbers */ // TYPE_NAME read by a type id that is a number on the wire, as String() renders the lookup (objects.js keys it by the // ids it knows; an id it does not know renders as the platform's missing value did). const typeText = (type: number): string => String(Reflect.get(TYPE_NAME, type)); // A git object as the pack parser and resolver hand it over (and as the harvest holds it). export type PackObject = { readonly type: number; readonly data: Uint8Array }; // ---- request body (pure) ---- type BadRequest = ErrorOf<'bad_request'>; const UNIT: Result = ok(null); // A count on the wire: a non-negative safe integer (a NaN or a fraction is refused, never written). const countField = (value: number, what: string): Result => { if (!Number.isSafeInteger(value) || value < 0) return fail('bad_request', `invalid ${what}: must be a non-negative safe integer`, { what }); return ok(value); }; // done is true, false or left out (the default, true). Anything else is refused: a null, 0 or '' from a JavaScript // caller used to leave `done` off the request while the answer was judged as the answer to a done request. const doneField = (value: unknown): Result => typeof value === 'boolean' ? ok(value) : fail('bad_request', 'invalid done: must be true, false or left out', { what: 'done' }); // deepen-since is a unix timestamp on the wire (a number from callers, or a digit string). Either way it is one token. const sinceField = (value: number | string, what: string): Result => typeof value === 'number' ? countField(value, what) : validateWire(value, what); // Every value of a list argument as one wire field, or the first that is not. function wireList(values: readonly string[], what: string): Result { const out: WireField[] = []; for (const v of values) { const valid = validateWire(v, what); if (!valid.ok) return valid; out.push(valid.value); } return ok(out); } // A want, a have and a shallow are object ids. upload-pack.c reads each with get_oid_hex and dies otherwise: parse_want // "expected to get oid, not 'want …'" (a ref name, an abbreviation, an empty want: ref-in-want is its own `want-ref` // line, which ply does not send), process_haves "expected SHA1 object, got '…'", process_shallow "invalid shallow line". // get_oid_hex reads either case (hexval), so an upper-case id is the same object to git: it is lowercased, as ls-refs // lowercases a listed oid, and parsed once. git reads only the first OID_HEX_CHARS characters and ignores the rest of the // line; ply refuses text after the id instead of sending it (test/want-oracle.test.js). A NUL, CR or LF is the wire // refusal it always was. type OidArg = 'want' | 'have' | 'shallow'; function oidList(values: readonly string[], what: OidArg): Result { const out: Oid[] = []; for (const v of values) { const field = validateWire(v, what); if (!field.ok) return field; const id = parseOid(field.value.toLowerCase()); if (!id.ok) return fail('bad_request', `invalid ${what}: must be an object id (${OID_HEX_CHARS} hex digits)`, { what }); out.push(id.value); } return ok(out); } // The arguments of one v2 fetch command. An empty filter, a zero deepen and a zero deepen-since are absent. export type FetchConfig = { readonly haves: readonly string[]; readonly filter: string; readonly deepen: number; readonly deepenSince: number | string; readonly deepenNot: readonly string[]; readonly deepenRelative: boolean; readonly thinPack: boolean; readonly ofsDelta: boolean; readonly includeTag: boolean; readonly noProgress: boolean; readonly done: boolean; readonly shallows: readonly string[]; readonly bypassCache: boolean; }; export type FetchOpts = { readonly wants: readonly string[] } & Input; const fetchDefaults = (): FetchConfig => ({ haves: [], filter: '', deepen: 0, deepenSince: 0, deepenNot: [], deepenRelative: false, thinPack: false, ofsDelta: true, includeTag: false, noProgress: true, done: true, shallows: [], bypassCache: false, }); // A fetch command parsed once (fetchRequest): every want, have and shallow an object id, every other token one wire field // and every count a checked integer, so encoding it (fetchRequestBody) checks nothing. No filter is null, and no deepen or // deepen-since is 0. export type FetchRequest = { readonly wants: readonly Oid[]; readonly haves: readonly Oid[]; readonly shallows: readonly Oid[]; readonly deepenNot: readonly WireField[]; readonly filter: WireField | null; readonly deepen: number; readonly deepenSince: number | WireField; readonly deepenRelative: boolean; readonly thinPack: boolean; readonly ofsDelta: boolean; readonly includeTag: boolean; readonly noProgress: boolean; readonly done: boolean; }; // The caller's options, checked in this order: wants, haves, shallows, deepen-nots, the filter, deepen, deepen-since, // done. The parsed `done` is the only one read afterwards: it decides the request's `done` line (fetchRequestBody) and // how the answer is judged (packStream, v2End). // An empty filter, a zero (or NaN) deepen and a falsy deepen-since are left out, as they always have been. // oxlint-disable-next-line complexity -- one check per fetch argument export function fetchRequest(opts: FetchOpts): Result { const { haves, filter, deepen, deepenSince, deepenNot, deepenRelative, thinPack, ofsDelta, includeTag, noProgress, done, shallows } = withDefaults(opts, fetchDefaults()); const w = oidList(opts.wants, 'want'); if (!w.ok) return w; const h = oidList(haves, 'have'); if (!h.ok) return h; const sh = oidList(shallows, 'shallow'); if (!sh.ok) return sh; const dn = wireList(deepenNot, 'deepen-not'); if (!dn.ok) return dn; let f: WireField | null = null; if (filter) { const c = validateWire(filter, 'filter'); if (!c.ok) return c; f = c.value; } let d = 0; if (deepen) { const c = countField(deepen, 'deepen'); if (!c.ok) return c; d = c.value; } let since: number | WireField = 0; // oxlint-disable-next-line typescript/strict-boolean-expressions -- 0, '' and NaN mean no deepen-since, as they always have if (deepenSince) { const c = sinceField(deepenSince, 'deepen-since'); if (!c.ok) return c; since = c.value; } const sentDone = doneField(done); if (!sentDone.ok) return sentDone; return ok({ wants: w.value, haves: h.value, shallows: sh.value, deepenNot: dn.value, filter: f, deepen: d, deepenSince: since, deepenRelative, thinPack, ofsDelta, includeTag, noProgress, done: sentDone.value }); } // The command=fetch body of a parsed request. // oxlint-disable-next-line complexity -- one optional line per fetch argument export function fetchRequestBody(req: FetchRequest): Result, BadRequest> { const args = [ ...req.wants.map(w => `want ${w}\n`), ...req.haves.map(h => `have ${h}\n`), ...req.shallows.map(s => `shallow ${s}\n`), ...(req.filter === null ? [] : [`filter ${req.filter}\n`]), ...(req.deepen ? [`deepen ${req.deepen}\n`] : []), // oxlint-disable-next-line typescript/strict-boolean-expressions -- 0 is no deepen-since; a parsed string is never empty ...(req.deepenSince ? [`deepen-since ${req.deepenSince}\n`] : []), ...req.deepenNot.map(r => `deepen-not ${r}\n`), ...(req.deepenRelative ? ['deepen-relative\n'] : []), ...(req.thinPack ? ['thin-pack\n'] : []), ...(req.ofsDelta ? ['ofs-delta\n'] : []), ...(req.includeTag ? ['include-tag\n'] : []), ...(req.noProgress ? ['no-progress\n'] : []), ...(req.done ? ['done\n'] : []), ]; return pkts(['command=fetch\n', ...args], { delimAfter: 0 }); } // The caller's options as a command=fetch body: parsed once, then encoded. export function fetchBody(opts: FetchOpts): Result, BadRequest> { const req = fetchRequest(opts); return req.ok ? fetchRequestBody(req.value) : req; } // ---- v2 fetch response state machine (pure step over frames) ---- // One sideband frame: the band byte and its payload. export type Sideband = { readonly band: number; readonly body: Uint8Array }; // @decoder parseSideband prefix=sideband export function parseSideband(payload: Uint8Array): Result> { // an empty payload (0004) has no band byte: git's demultiplex_sideband "missing sideband designator" // take returns exactly SIDEBAND_CHANNEL_BYTES bytes or the short read, so the fallback never applies const head = take(payload, 0, SIDEBAND_CHANNEL_BYTES, 'protocol'); if (!head.ok) return head; const band: number = atOr(head.value, 0, 0); const body = payload.subarray(SIDEBAND_CHANNEL_BYTES); return ok({ band, body }); } // The response's two phases: section lines up to `packfile`, then sideband frames. `section` is the open section // (null between a delim and the next header); its line count is `sectionLines` (shallow-info) or `otherLines` (a // section ply does not read). `done` is the flush that ends the response, in either phase. export type V2Phase = 'sections' | 'pack'; export type V2State = { readonly limits: CheckedLimits; readonly phase: V2Phase; readonly section: string | null; readonly sections: readonly string[]; readonly shallow: readonly string[]; readonly unshallow: readonly string[]; readonly acks: readonly string[]; readonly ready: boolean; readonly band1: number; readonly band2: number; readonly band3: number; readonly band2Frames: number; readonly progress: string; readonly progressCarry: Uint8Array; readonly error: null; readonly done: boolean; readonly packFrames: number; readonly sectionLines: number; readonly otherLines: number; }; export type V2Step = { readonly state: V2State; readonly pack: Uint8Array | null }; // upload-pack's own words for an oid it will not send. v2 prefixes `upload-pack: `; v0 and v1 do not. A trailing oid // is separated by a space, so `not our refresh` is a different error. Recognised once, here (#112): callers switch // on `reason` instead of the message (#76). Any other text carries no reason (an empty extra adds no field). const NOT_OUR_REF = 'not our ref'; const UPLOAD_PACK_NOT_OUR_REF = 'upload-pack: not our ref'; function notOurRefReason(text: string): Extra { const bare = text === NOT_OUR_REF || text.startsWith(`${NOT_OUR_REF} `); const prefixed = text === UPLOAD_PACK_NOT_OUR_REF || text.startsWith(`${UPLOAD_PACK_NOT_OUR_REF} `); if (bare || prefixed) return { reason: 'not_our_ref' }; return {}; } // The sideband counters a demultiplexer keeps (V2State carries them): bytes and frames per band, and the progress tail. export type SidebandCounters = { readonly band1: number; readonly band2: number; readonly band2Frames: number; readonly packFrames: number; readonly progress: string; readonly progressCarry: Uint8Array; }; // What one frame was: pack bytes (a view into the frame's payload, which pktStep copied out of the chunk), or progress. export type SidebandPiece = { readonly kind: 'pack'; readonly body: Uint8Array } | { readonly kind: 'progress'; readonly text: string }; export type Sidebanded = { readonly state: S; readonly piece: SidebandPiece }; export type SidebandError = ErrorOf<'protocol'> | ErrorOf<'limit_exceeded'> | ErrorOf<'remote_error'>; // One packfile-section frame, demultiplexed (git's demultiplex_sideband): band 1 is pack bytes and band 2 progress, each // held to maxPack before the frame is handed on; band 3 is the remote's error, its text the message. Peak is the frame. // scrub: the transport's credential scrubber (or null), applied to band 3's text before it is cut (errors.ts remoteText). // @decoder sidebandStep prefix=sideband export function sidebandStep(st: S, frame: Frame, maxPack: number, scrub: Scrub | null = null): Result, SidebandError> { // A delim or response-end inside the packfile section ends git's sideband demultiplexer (demultiplex_sideband: // SIDEBAND_FLUSH), so git reads a pack cut there; neither is a frame upload-pack writes in it. Refused, not skipped. if (frame.kind !== 'data') return proto(`v2: ${frame.kind} inside the packfile section`); const sb = parseSideband(frame.payload); if (!sb.ok) return sb; const band = sb.value; if (band.band === SIDEBAND_PACK) { const band1 = st.band1 + band.body.length; if (band1 > maxPack) return err(limitExceeded('pack bytes', band1, maxPack)); return ok({ state: { ...st, band1, packFrames: st.packFrames + 1 }, piece: { kind: 'pack', body: band.body } }); } if (band.band === SIDEBAND_PROGRESS) { const band2 = st.band2 + band.body.length; if (band2 > maxPack) return err(limitExceeded('progress bytes', band2, maxPack)); // decodeChunk: a character cut by a frame boundary is held for the next frame instead of decoding as two U+FFFD const d = decodeChunk(st.progressCarry, band.body); return ok({ state: { ...st, band2, band2Frames: st.band2Frames + 1, progress: (st.progress + d.text).slice(-PROGRESS_KEEP), progressCarry: d.carry }, piece: { kind: 'progress', text: d.text } }); } if (band.band === SIDEBAND_ERROR) { const text = td.decode(band.body).trim(); return fail('remote_error', `remote: ${remoteText(text, REMOTE_TEXT_MAX, scrub)}`, notOurRefReason(text)); } return proto(`v2: bad sideband ${band.band}`); } // A commit want whose object is a tree, blob, or tag is not_found, before pack_corrupt is built. Any other mismatch // (a blob want, a delta, an unknown type id) stays the caller's pack_corrupt. function refuseNonCommit(expectType: number | null, type: number, oid: string): Result> { if (expectType !== TYPE.COMMIT) return UNIT; const found = nonCommitType(type); switch (found.tag) { case 'commit': case 'delta': case 'invalid': return UNIT; case 'notCommit': return fail('not_found', `${oid} is not a commit`, { miss: { kind: 'commit', miss: { kind: 'not_commit', oid, type: found.name } } }); } // Stryker disable next-line ConditionalExpression: unreachable, the compiler's exhaustiveness check return assertNever(found); } // characters of a refused acknowledgment line quoted in the error (remoteQuoted: cut first, then every control and format character escaped) const ACK_LINE_SHOWN = 60; // v2End's "no packfile section" names the answer's sections. A section header is a line the server wrote: each name // goes through remoteText cut to SECTION_SHOWN units, and at most SECTIONS_SHOWN of them are named. const SECTIONS_SHOWN = 8; const SECTION_SHOWN = 60; // limits: a checked record (parseLimits; DEFAULT_LIMITS when none is given): v2Step reads its caps on every frame. export const v2Init = (limits: CheckedLimits = DEFAULT_LIMITS): V2State => Object.freeze({ limits, phase: 'sections', section: null, sections: [], shallow: [], unshallow: [], acks: [], ready: false, band1: 0, band2: 0, band3: 0, band2Frames: 0, progress: '', progressCarry: new Uint8Array(0), error: null, done: false, packFrames: 0, sectionLines: 0, otherLines: 0, }); const stepped = (state: V2State, pack: Uint8Array | null = null): Result => ok({ state, pack }); // scrub: the transport's credential scrubber (or null), applied to an ERR line's or band 3's text before it is cut. // @decoder v2Step prefix=v2 // oxlint-disable-next-line complexity -- a step function is one transition table export function v2Step(st: V2State, frame: Frame, scrub: Scrub | null = null): Result { // Every section line, including ones this client does not interpret, counts against the same cap. The state's limits // are checked (v2Init takes a CheckedLimits): a missing maxSectionLines used to make `n >= ` false. const maxLines = st.limits.maxSectionLines; // The one place every consumer of a fetch response passes: band 1 (the pack) and band 2 (progress) are each held to // maxPackBytes here, before the frame is yielded, so fetchSingle, fetchWide, fetchPackOnce, the streamed blob (edit, // read, checkout) and any new consumer are bounded without a check of their own. Before, only the buffered consumers // counted, and a streamed blob body was accepted until the request timer (~5 GiB at the default 60 s). const maxPack = st.limits.maxPackBytes; if (st.done) return proto('v2: frame after flush'); if (frame.kind === 'flush') return stepped({ ...st, done: true }); switch (st.phase) { case 'pack': { const sb = sidebandStep(st, frame, maxPack, scrub); if (!sb.ok) return sb; // Stryker disable next-line ConditionalExpression: a progress piece has no body field, and stepped's default parameter makes the missing value the same null return stepped(sb.value.state, sb.value.piece.kind === 'pack' ? sb.value.piece.body : null); } case 'sections': { // A response-end outside the packfile section is skipped. if (frame.kind === 'end') return stepped(st); if (frame.kind === 'delim') return stepped({ ...st, section: null }); const line = td.decode(frame.payload).replace(/\n$/, ''); if (line.startsWith('ERR ')) { const text = line.slice('ERR '.length); return fail('remote_error', `remote: ${remoteText(text, REMOTE_TEXT_MAX, scrub)}`, notOurRefReason(text)); } if (st.section === null) { if (st.sections.length >= maxLines) return err(limitExceeded('section headers', st.sections.length + 1, maxLines)); const next: { -readonly [K in keyof V2State]: V2State[K] } = { ...st, section: line, sections: [...st.sections, line] }; if (line === 'packfile') next.phase = 'pack'; return stepped(next); } if (st.section === 'shallow-info') { if (st.sectionLines >= maxLines) return err(limitExceeded('shallow-info lines', st.sectionLines + 1, maxLines)); const s = { ...st, sectionLines: st.sectionLines + 1 }; // the server may repeat a shallow line when a commit is wanted both directly and through a tag; keep them unique if (line.startsWith('shallow ')) return stepped({ ...s, shallow: s.shallow.includes(line.slice('shallow '.length)) ? s.shallow : [...s.shallow, line.slice('shallow '.length)] }); if (line.startsWith('unshallow ')) return stepped({ ...s, unshallow: s.unshallow.includes(line.slice('unshallow '.length)) ? s.unshallow : [...s.unshallow, line.slice('unshallow '.length)] }); return stepped(s); } if (st.section === 'acknowledgments') { if (st.acks.length >= maxLines) return err(limitExceeded('acknowledgment lines', st.acks.length + 1, maxLines)); if (line === 'ready') return stepped({ ...st, ready: true }); // git reads NAK, ACK and ready, and dies on anything else ("unexpected acknowledgment line") if (line !== 'NAK' && !line.startsWith('ACK ')) return proto(`v2: unexpected acknowledgment line ${remoteQuoted(line, ACK_LINE_SHOWN)}`); return stepped({ ...st, acks: [...st.acks, line] }); } if (st.otherLines >= maxLines) return err(limitExceeded('section lines', st.otherLines + 1, maxLines)); return stepped({ ...st, otherLines: st.otherLines + 1 }); } } // Stryker disable next-line ConditionalExpression: unreachable, the compiler's exhaustiveness check return assertNever(st.phase); } // The response is over: it ended with its flush, and it is the answer the request asked for. git is the judge of its // shape. `done` is whether the request carried `done` (the parsed request's, fetchRequest): after it git's fetch-pack // reads the packfile section next and dies on an acknowledgments section ("expected 'packfile', received // 'acknowledgments'"), so one is refused here whether or not a pack follows it, and an answer with no pack is never ok. // Without `done` a round may end on its acknowledgments, unless the server said `ready`: the packfile follows that in // the same response (git: "expected packfile to be sent after 'ready'"). // A response with no section at all, not even its flush, carries reason 'empty_response': the answer git's upload-pack // gives when it dies on an argument (fetch.ts filterUnadvertised reads it). export function v2End(st: V2State, done: boolean): Result> { if (!st.done) return fail('protocol', 'v2: response ended without flush', st.sections.length === 0 ? { reason: 'empty_response' } : {}); const acks = st.sections.includes('acknowledgments'); if (st.phase === 'pack') return done && acks ? proto('v2: expected packfile, received acknowledgments') : ok(st); if (done || !acks) return proto(`v2: no packfile section (sections: ${st.sections.slice(0, SECTIONS_SHOWN).map(s => remoteText(s, SECTION_SHOWN)).join(',') || 'none'})`); if (st.ready) return proto('v2: the server said ready and sent no packfile section'); return ok(st); } export const v2Attrs = (st: V2State): Attrs => ({ 'git.fetch.sections': st.sections.join(','), 'git.fetch.shallow': st.shallow, 'git.fetch.unshallow': st.unshallow, 'git.fetch.acks': st.acks, 'git.fetch.ready': st.ready, 'git.sideband.band1.bytes': st.band1, 'git.sideband.band2.bytes': st.band2, 'git.sideband.band2.frames': st.band2Frames, 'git.sideband.band3.bytes': st.band3, 'git.sideband.pack.frames': st.packFrames, 'git.sideband.progress.tail': st.progress.slice(-PROGRESS_ATTR), }); // The request's arguments as the caller gave them: a left-out or null argument reads as its default. export const fetchAttrs = (opts: FetchOpts): Attrs => ({ 'git.command': 'fetch', 'git.protocol.version': PROTOCOL_V2, 'git.fetch.wants': opts.wants, 'git.fetch.wants.count': opts.wants.length, 'git.fetch.haves.count': getOrElse(fromNullish(opts.haves), []).length, 'git.fetch.filter': getOrElse(fromNullish(opts.filter), ''), 'git.fetch.deepen': getOrElse(fromNullish(opts.deepen), 0), 'git.fetch.deepen_since': getOrElse(fromNullish(opts.deepenSince), 0), 'git.fetch.deepen_not': getOrElse(fromNullish(opts.deepenNot), []), 'git.fetch.shallows.count': getOrElse(fromNullish(opts.shallows), []).length, 'git.fetch.done': opts.done !== false, 'git.fetch.thin_pack': Boolean(opts.thinPack), 'git.fetch.ofs_delta': opts.ofsDelta !== false, 'git.fetch.no_progress': opts.noProgress !== false, }); // Band-1 pack bytes as they arrive; the v2 state is the generator's return value. export type PackFetch = AsyncGenerator; // The response read frame by frame: band-1 bytes yielded as they arrive, the v2 state or the first Err returned. A // caller's own span (GitHttp.packStream) may not carry t.scrub: band-3/ERR text is redacted here too, and so is // whatever the transport's stream raised (it is re-raised, scrubbed). // The request is parsed once here: its `done` is the line that is sent and the one the answer is judged against (v2End). async function* v2Frames(t: Transport, opts: FetchOpts, span: Span): AsyncGenerator, unknown> { const req = fetchRequest(opts); if (!req.ok) return req; const body = fetchRequestBody(req.value); if (!body.ok) return body; // the exchange (no request yet: the frames are pulled below), then the limits it runs under const exchange = frames(t, 'git-upload-pack', body.value, span, { bypassCache: Boolean(opts.bypassCache) }); let st = v2Init(t.limits); try { for await (const f of exchange) { const r = v2Step(st, f, t.scrub); if (!r.ok) { scrubError(r.error, t.scrub); return r; } st = r.value.state; if (r.value.pack) yield r.value.pack; } } catch (e) { throw scrubError(e, t.scrub); } return v2End(st, req.value.done); } // Stream band-1 pack bytes. Consumer drives it; the v2 state is the generator's return value. A stream: a failure is // raised through it (the iterator rejects with the PlyError). export async function* packStream(t: Transport, opts: FetchOpts, span: Span): PackFetch { span.set(fetchAttrs(opts)); const end = yield* v2Frames(t, opts, span); if (!end.ok) throw end.error; span.set(v2Attrs(end.value)); return end.value; } // ---- single-object pack stream parser (pure step) ---- // Emits the zlib bytes of the single object, withholding the trailing 20-byte trailer, verifying it at finish. // A bounded read that ran past the buffer (`reason: 'short_read'`, bytes.ts). spStep waits for more bytes on this reason, // never on the message (#76). A finished header readObjHeader refuses (a varint past 64 bits) is that error, returned. export const isShortRead = (e: unknown): boolean => thrownCodeIs(e, 'pack_corrupt') && thrownReasonIs(e, 'short_read'); // The single-object parser's state: the pack header and the object header (`header`), then the zlib body with the // trailer withheld (`body`); `trailer` is null until spFinish has checked it, then the trailer it checked. type SpCommon = { readonly head: Uint8Array; readonly lag: Uint8Array; readonly sha: Sha1; readonly bytes: number; readonly zlibBytes: number; readonly trailer: string | null; readonly maxObject: number }; export type SpHeader = SpCommon & { readonly phase: 'header'; readonly type: number; readonly size: number }; export type SpBody = SpCommon & { readonly phase: 'body'; readonly type: ObjType; readonly size: ObjSize; readonly headerBytes: number }; export type SpState = SpHeader | SpBody; export type SpFinished = SpBody & { readonly trailer: string }; export type SpStepped = { readonly state: SpState; readonly zlib: Uint8Array[] }; type Corrupt = ErrorOf<'pack_corrupt'>; type SpError = HeaderError; // `maxObject`: the cap on the object's declared size, refused where its header is read (limit_exceeded, `pack object // size`). A streamed blob (edit, read) has none: its reader bounds it. export const spInit = (maxObject: number = NO_SIZE_CAP): SpState => Object.freeze({ head: new Uint8Array(0), phase: 'header', type: 0, size: 0, lag: new Uint8Array(0), sha: new Sha1(), bytes: 0, zlibBytes: 0, trailer: null, maxObject }); const hasSignature = (head: Uint8Array): boolean => head.subarray(0, PACK_SIGNATURE.length).every((b, i) => b === PACK_SIGNATURE[i]); // The header phase, given every byte so far (`head`): still waiting (null), or the body state and the bytes after the // object header. Past the length check the header fields are views into `head`, not bounded reads: they cannot be short. function spHeader(s: SpHeader, head: Uint8Array): Result<{ readonly body: SpBody; readonly rest: Uint8Array } | null, SpError> { // need the pack header plus the first object-header byte; a short read means the chunk has not arrived. if (head.length < PACK_HEADER_BYTES + 1) return ok(null); if (!hasSignature(head)) return fail('pack_corrupt', 'pack: bad signature'); const version = readUint32BE(head, PACK_SIGNATURE.length); if (!version.ok) return version; if (version.value !== PACK_VERSION) return fail('pack_corrupt', 'pack: bad version'); const counted = readUint32BE(head, PACK_SIGNATURE.length + UINT32_BYTES); if (!counted.ok) return counted; const count = counted.value; if (count !== 1) return fail('pack_corrupt', `pack: expected 1 object, got ${count}`); const header = readObjHeader(head, PACK_HEADER_BYTES, PACK_HEADER_BYTES, s.maxObject); if (!header.ok) return isShortRead(header.error) ? ok(null) : header; if (header.value.type === TYPE.OFS_DELTA || header.value.type === TYPE.REF_DELTA) return fail('pack_corrupt', 'pack: single-object pack is a delta'); const p: number = header.value.next; s.sha.update(head.subarray(0, p)); return ok({ body: { ...s, phase: 'body', type: header.value.type, size: header.value.size, head: new Uint8Array(0), headerBytes: p }, rest: head.subarray(p) }); } // Where a chunk lands: still in the header (the state waits for more), or body bytes after the object header. type SpEntry = { readonly tag: 'wait'; readonly state: SpHeader } | { readonly tag: 'body'; readonly state: SpBody; readonly data: Uint8Array }; function spEnter(st: SpState, chunk: Uint8Array): Result { switch (st.phase) { case 'header': { // The header phase is bounded here, for every caller: readObjHeader refuses an object header past 10 bytes // (VARINT_MAX_SHIFT), so `head` never holds more than PACK_HEADER_BYTES + 9 bytes between chunks and the 22nd // byte of a pack whose object header never ends is `pack_corrupt` (#13: endless 0x80 used to grow `head` without bound). const head = concat([st.head, chunk]); const opened = spHeader(st, head); if (!opened.ok) return opened; if (opened.value === null) return ok({ tag: 'wait', state: { ...st, head } }); return ok({ tag: 'body', state: opened.value.body, data: opened.value.rest }); } case 'body': return ok({ tag: 'body', state: st, data: chunk }); } // Stryker disable next-line ConditionalExpression: unreachable, the compiler's exhaustiveness check return assertNever(st); } // @decoder spStep prefix=sp-header export function spStep(st: SpState, chunk: Uint8Array): Result { const entered = spEnter({ ...st, bytes: st.bytes + chunk.length }, chunk); if (!entered.ok) return entered; const entry = entered.value; if (entry.tag === 'wait') return ok({ state: entry.state, zlib: [] }); const { state: s, data } = entry; // body: pass through with the trailer withheld // Stryker disable next-line ConditionalExpression,EqualityOperator: equivalent: concat([empty lag, data]) is a copy of data, byte for byte const all = s.lag.length > 0 ? concat([s.lag, data]) : data; const emitLen = Math.max(0, all.length - PACK_TRAILER_BYTES); const emit = all.subarray(0, emitLen); const lag = all.slice(emitLen); // The zlib bytes of an object declared `size` may not run past zlibBound(size) (format.js). The declared-size check on // inflated bytes cannot see a stream that inflates to nothing (non-final empty stored blocks, 5 bytes in, 0 out), which // used to be accepted until the request timer; this refuses it on the chunk that crosses the bound, so a 10-byte blob // costs at most 26 zlib bytes plus the rest of that chunk, whatever maxPackBytes is. The message carries no count: the // same bytes in any chunking give the same error (I10). const zlibBytes = s.zlibBytes + emitLen; const bound = zlibBound(s.size); // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- an ad-hoc error field (`size`); Extra has got and max if (zlibBytes > bound) return fail('pack_corrupt', `pack: zlib stream longer than its declared size allows (${s.size}-byte object, ${bound}-byte bound)`, { got: zlibBytes, max: bound, size: s.size } as Extra); // Stryker disable next-line ConditionalExpression: hashing an empty emit leaves the trailer digest unchanged if (emitLen) s.sha.update(emit); return ok({ state: { ...s, lag, zlibBytes }, zlib: emitLen ? [emit] : [] }); } export function spFinish(st: SpState): Result { if (st.phase !== 'body') return fail('pack_corrupt', 'pack: truncated before object header'); if (st.lag.length !== PACK_TRAILER_BYTES) return fail('pack_corrupt', 'pack: missing trailer'); const got = st.sha.hex(), want = toHex(st.lag); if (want !== got) return fail('pack_corrupt', `pack: trailer ${want} != ${got}`); return ok({ ...st, trailer: want }); } const trailerOf = (st: SpState): string => (st.trailer === null ? '' : st.trailer); export const spAttrs = (st: SpState, prefix = 'git.pack'): Attrs => ({ [`${prefix}.objects`]: 1, [`${prefix}.bytes`]: st.bytes, [`${prefix}.trailer`]: trailerOf(st), [`${prefix}.compressed.bytes`]: st.zlibBytes, [`${prefix}.type.${typeText(st.type)}`]: 1 }); // One object by oid (a tree or a commit, under a filter that keeps the pack to that object), fully in memory but bounded // by limits.maxObjectSize, inflated with our inflate and hash-verified against the oid. A verification failure after a // cache HIT is retried once past the cache (packcache.js). Returns { type, data, oid, size, bust }. // // The blob:none fallback. Some servers advertise `fetch=filter` but honour only `blob:none` (Cloudflare Artifacts: // HTTP 400 "Unsupported Git fetch filter; only blob:none is supported.", observed 2026-10-23). Detected by that answer, // never by host name: the transport's `quirks.treeFilter` becomes 'blob:none' and this and every later tree:0 fetch on // it asks for blob:none instead. That pack is wider — a commit want (deepen 1) brings every tree of the commit, a tree // want its whole subtree — so it is parsed and delta-resolved (packresolve.js), the wanted object is taken by its // computed hash, and the other commits and trees are harvested into `t.objects` (bounded by limits.maxHarvestBytes): // the next tree:0 fetch of any of them is answered locally. Spans: `git.fetch.filter.fallback`, `git.fetch.wide.*`. // The phrases a server uses when it refuses a fetch filter. Cloudflare Artifacts (observed 2026-10-23) is HTTP 400 // plus the first one; git's upload-pack says "filter '' not supported" (upload-pack.c). One predicate, shared: // ply-web used to keep a copy of it next to the cache. The text is remote, so it is read in linear time. A single // `filter .* not` restarts its run at every "filter " (quadratic, ply#274); here each "filter " takes the rest of its // line once, the global exec resumes after it, and one fixed-phrase test (a bounded number of steps at each start) // reads that rest. The first "filter " on a line takes the longest rest, so a later one on the same line could find // nothing it did not. Same flags as before (/i, no /u): the same case fold. const FILTER_PHRASE = /only blob:none is supported|unsupported (git )?fetch filter/i; const FILTER_REST = /filter ([^\n\r\u2028\u2029]*)/gi; const FILTER_NOT = / not (supported|allowed)/i; const isFilterPhrase = (text: string): boolean => { if (FILTER_PHRASE.test(text)) return true; FILTER_REST.lastIndex = 0; // shared /g: every call starts at the beginning, whatever an earlier return left for (let m = FILTER_REST.exec(text); m !== null; m = FILTER_REST.exec(text)) if (FILTER_NOT.test(atOr(m, 1, ''))) return true; return false; }; // oxlint-disable-next-line no-magic-numbers -- RFC 9110 Bad Request: Artifacts' refusal, and the only status this predicate accepts const FILTER_REFUSAL_STATUS = 400; // A raw upload-pack answer (the HTTP status and the body, or the text an error quoted from it). Status is part of the // answer: a 500 that happens to contain the phrase is not a filter refusal. export const isFilterRefusal = (status: number, text: string): boolean => status === FILTER_REFUSAL_STATUS && isFilterPhrase(text); // A thrown value is not a PlyError until it is: message may be no string, and `status` is only on an http error. export const isFilterRejection = (e: unknown): boolean => { const code = getOrElse(thrownCode(e), null); const text = String(getOrElse(thrownMessage(e), '')); // remote_error and protocol have no HTTP status. The phrase in the message is the refusal, so the same predicate // sees it as the 400 it is when the server says it over HTTP. if (code === 'remote_error' || code === 'protocol') return isFilterRefusal(FILTER_REFUSAL_STATUS, text); if (code !== 'http') return false; if (e === null || typeof e !== 'object' || !Object.hasOwn(e, 'status')) return false; const status: unknown = Reflect.get(e, 'status'); return typeof status === 'number' && isFilterRefusal(status, text); }; // The no-filter fallback. git's upload-pack takes a `filter` argument only with uploadpack.allowFilter set, which is not // git's default: a stock `git http-backend` does not list `filter` on its `fetch=` line, and on a request that sends one // it dies without a word (upload-pack.c process_args: "unexpected line"), no ERR, no flush, an empty body. That answer // to a filtered request (v2End's reason 'empty_response'), and only that one, sends ply to the advertisement, the one // place the server says what `fetch` takes (GET info/refs, read as discover reads it). No `filter` there: the transport's // quirks.treeFilter becomes 'none' (learnFilter), true is returned, and the caller asks again without a filter, taking // the wider pack (a commit want with deepen 1 is its whole snapshot, a tree want its whole subtree; held to every pack // limit like any other). `filter` advertised, or an advertisement that cannot be read: false, and the empty response // stays the protocol error it is. An advertisement already in hand (discover) sets the quirk before any filtered fetch. // Spans: this one is `git.upload-pack discover` with git.server.fetch/filter; the fetch that follows carries // `git.fetch.filter.fallback: 'none'`. export const isEmptyResponse = (e: unknown): boolean => thrownCodeIs(e, 'protocol') && thrownReasonIs(e, 'empty_response'); // The other ways a server refuses an argument it does not take: an ERR pkt-line (remote_error) or HTTP 400 (Artifacts' // answer to a filter it does not take). 401, 403, 404, 429 and 5xx are other codes and never match. objectAt reads the // advertisement on one of these only for a `deepen` it added itself, once per quirks record. const isArgumentRefusal = (e: PlyError): boolean => e.code === 'remote_error' || (e.code === 'http' && e.status === FILTER_REFUSAL_STATUS); // `refusal`: the answers that send this call to the advertisement. Every caller but objectAt's depth takes the default. export async function filterUnadvertised(t: Transport, e: PlyError, refusal: (e: PlyError) => boolean = isEmptyResponse): Promise { if (!refusal(e)) return false; const attrs = { 'git.command': 'discover', 'git.protocol.version': PROTOCOL_V2 }; // oxlint-disable-next-line typescript/promise-function-async -- the exchange's promise, passed through const read = await coded('protocol', () => spanned(t, 'git.upload-pack discover', clientSpan(t, attrs), async (span: Span): Promise> => { const buf = await advertise(t, 'git-upload-pack', span); if (!buf.ok) return buf; const framed = pktStep(pktInit(), buf.value); if (!framed.ok) return framed; const finished = pktFinish(framed.value.state); if (!finished.ok) return finished; const lines = parseCapabilitiesV2(framed.value.frames, t.scrub); if (!lines.ok) return lines; const features = capLookup(lines.value).features('fetch'); const filter = features.includes('filter'); const shallow = features.includes('shallow'); span.set({ ...pktAttrs(finished.value), 'git.server.fetch': features, 'git.server.filter': filter, 'git.server.shallow': shallow }); return ok({ filter, shallow }); })); if (!read.ok) return false; learnFilter(t, read.value.filter); // the same line says whether the server takes `deepen`: objectAt asks again without the depth it added learnShallow(t, read.value.shallow); return !read.value.filter; } // Check a wide (blob:none) pack's commits, trees and tags before any of it is returned or harvested (#18, #46). // `wanted(oid, o)` names what a tree:0 server would have sent — what the caller asked for. Those pass assertFetched // under the caller's limits exactly as a single-object fetch would, and any failure is the fetch's. The rest are extras // only the widening brought, and the two ways one can fail are treated differently: // - limit_exceeded (say a sibling directory with more than maxTreeEntries entries): the extra is dropped from the // pack, so it is neither returned nor harvested, and counted. A limit is the caller's bound on what it will parse for // what it reads; an honest repository can hold a tree past it that the caller never reads, and a tree:0 server would // never have sent it. Failing on it made the answer depend on the server's filter support (#46: one 2,135-entry // directory failed every read of curl on Artifacts). Dropping it is safe: the caller never sees it, and a later // want of it is a fetch of its own that applies the limits to it as the wanted object. Whether its bytes past the // limit are well formed is unobservable, since checking them would mean parsing past the limit. // - pack_corrupt (malformed; git fsck would refuse it): the fetch fails, as #18 decided. This server sent bytes no // honest git server holds, hash-verified or not, so nothing from this response is trusted, including its other // objects; pack_corrupt is retryable (and a cached response is refetched once past the cache). // maxPackBytes, maxPackObjects and maxInflatedTotal have already bounded the whole pack. Returns the skipped extras, // or the first failure that is not one (nothing is dropped then). export type Skipped = { readonly oid: string; readonly what: unknown; readonly bytes: number }; export function checkWide(objects: Map, wanted: (oid: string, o: PackObject) => boolean, limits: CheckedLimits): Result { const skipped: Skipped[] = []; for (const [oid, o] of objects) { if (o.type !== TYPE.COMMIT && o.type !== TYPE.TREE && o.type !== TYPE.TAG) continue; const checked = assertFetched(typeText(o.type), o.data, limits); if (checked.ok) continue; if (checked.error.code !== 'limit_exceeded' || wanted(oid, o)) return checked; // a limit_exceeded error carries `what` (limits.ts) skipped.push({ oid, what: checked.error.what, bytes: o.data.length }); } for (const s of skipped) objects.delete(s.oid); return ok(skipped); } export const skippedAttrs = (skipped: readonly Skipped[]): Attrs => ({ 'git.fetch.wide.skipped': skipped.length, 'git.fetch.wide.skipped_reason': [...new Set(skipped.map(s => s.what))].join(',') }); // Keep commits and trees from a wide pack for later tree:0 wants. Every entry's key is its computed hash (resolvePack); // every caller has run checkWide over the pack first (what is left passed assertFetched under the transport's limits), // since fetchObject serves the harvest without another look. export function harvest(t: Transport, objects: ReadonlyMap, skip: string | null = null): Result<{ readonly kept: number; readonly bytes: number }, PlyError> { const store = t.objects; const cap = t.limits.maxHarvestBytes; let bytes = 0, held = harvestBytes(store), kept = 0; for (const [oid, o] of objects) { if (oid === skip || store.has(oid) || (o.type !== TYPE.COMMIT && o.type !== TYPE.TREE)) continue; if (held + o.data.length > cap) break; store.set(oid, { type: o.type, data: o.data }); held += o.data.length; kept++; bytes += o.data.length; } return ok({ kept, bytes }); } // A failed fetch's error carries the cache status of the response it came from (fetchPack and the retry read it back). export function stampCacheStatus(e: unknown, span: Span): void { if (e === null || typeof e !== 'object') return; Reflect.set(e, 'cacheStatus', getOrElse(fromNullish(span.attrs['git.response.cache']), '')); } // One upload-pack fetch under its span. What the stream raised is caught (coded: an uncoded value is pack_corrupt), and // a failure is stamped with the response's cache status for the retry rule (corruptCacheHit). export async function fetchExchange(t: Transport, attrs: Attrs, fn: (span: Span) => Promise>): Promise> { const held: { span: Span | null } = { span: null }; // oxlint-disable-next-line typescript/promise-function-async -- the exchange's promise, passed through (an async wrapper adds bytes, not behaviour) const r = await coded('pack_corrupt', () => spanned(t, 'git.upload-pack fetch', clientSpan(t, attrs), (span: Span): Promise> => { held.span = span; return fn(span); })); if (!r.ok && held.span !== null) stampCacheStatus(r.error, held.span); return r; } // A verification failure after a cache HIT is retried once past the cache (packcache.ts); a corrupt live response is not. async function onceMore(once: (bust: boolean) => Promise>): Promise> { const first = await once(false); if (first.ok || !corruptCacheHit(first.error)) return first; return once(true); } // One object wanted by oid. `expectType` (a TYPE id) is checked against what arrives; null takes any type. // `deepen`: the depth sent with the want. Left out (or null) it is 1 for a want that may be a commit (expectType null, // commit or tag) on a server that takes `deepen`, so one object costs one object of pack whatever the history behind it: // with no depth a commit want is answered with every commit reachable from it. A number is sent as given: n for the // commit and n - 1 generations of ancestors, 0 for no `deepen` line (the whole history). A want the caller names a tree // or a blob gets none: no history is reachable from one. export type ObjectWantConfig = { readonly filter: string; readonly deepen: number | null; readonly expectType: number | null }; export type ObjectWant = { readonly oid: string } & Input; const objectWantDefaults = (): ObjectWantConfig => ({ filter: 'tree:0', deepen: null, expectType: null }); // The entries of a fetched tree as its check (assertFetched) parsed them, and the entry cap they were parsed under: the // only limit parseTree reads (#208). export type CheckedTree = { readonly entries: TreeEntry[]; readonly maxTreeEntries: number }; // `tree`: a tree fetched by its own want carries the parse that checked it; a harvested or wide answer, and any other // type, none. type FetchedBase = { readonly type: number; readonly data: Uint8Array; readonly oid: Oid; readonly size: number; readonly bust: boolean; readonly tree: Option }; // A fetched object; `harvested` when the transport's harvest answered it, `wide` when a blob:none pack did. export type FetchedObject = FetchedBase | (FetchedBase & { readonly harvested: true }) | (FetchedBase & { readonly wide: true }); // A fetched tree's entries, parsed once (#208): the check's parse when the reader's entry cap is the one it was made // under (parseTree is a pure function of the bytes and that cap, so it is the parse the reader would make), else the // reader's own parse under its limits. The fetch hands its object to one reader, so the array is that reader's. export function treeEntries(fetched: FetchedObject, limits: CheckedLimits): Result { const { tree } = fetched; if (tree.tag === 'some' && tree.value.maxTreeEntries === limits.maxTreeEntries) return ok(tree.value.entries); return parseTree(fetched.data, limits); } type Single = { readonly oid: Oid; readonly filter: string; readonly deepen: number; readonly expectType: number | null }; // The object count in a pack header, or null when the header is not a version-2 pack (the single-object parser then // reports its own error). A commit want may be a tag plus the commit it peels to: more than one object, and still the // wanted object's type. function declaredPackCount(pack: Uint8Array): number | null { if (pack.length < PACK_HEADER_BYTES) return null; if (!hasSignature(pack)) return null; const version = readUint32BE(pack, PACK_SIGNATURE.length); if (!version.ok || version.value !== PACK_VERSION) return null; const count = readUint32BE(pack, PACK_SIGNATURE.length + UINT32_BYTES); return count.ok ? count.value : null; } // A multi-object answer to a commit want: the wanted object, once its type is a commit. A tree, blob, or tag is // not_found from refuseNonCommit. Null means the single-object parser should refuse the pack (it does not hold the want). function commitFromPack(t: Transport, oid: Oid, parsed: ParsedPack, span: Span, bust: boolean): Result { // Stryker disable next-line ArrowFunction: null and a bare return both mean the base is not in the pack const resolved = resolvePack(parsed, () => null, t.limits); if (!resolved.ok) return resolved; const objects: Map = resolved.value.objects; const found = mapGet(objects, oid); if (found.tag === 'none') return ok(null); const want = found.value; span.set({ 'git.pack.objects': parsed.stats.count, 'git.object.oid': oid, 'git.object.type': typeText(want.type) }); const refused = refuseNonCommit(TYPE.COMMIT, want.type, oid); if (!refused.ok) return refused; // a resolved object is commit, tree, blob or tag, and refuseNonCommit answered the other three // Stryker disable next-line ConditionalExpression,BlockStatement: see above if (want.type !== TYPE.COMMIT) return ok(null); const checked = assertFetched('commit', want.data, t.limits); if (!checked.ok) return checked; return ok({ type: want.type, data: want.data, oid, size: want.data.length, bust, tree: none }); } // A pack read through the collector as its chunks arrive (#172): the bytes are dropped once fed, so what is held is the // objects the collector keeps. A refusal is held, the rest of the response read and dropped, and then returned: the // response's own refusals (packStream raises them) come first, as when the whole pack was read before it was parsed. export type WirePack = { readonly state: V2State; readonly pack: Result }; export async function collectWire(it: PackFetch, c: PackCollector): Promise { let held: Result = UNIT; for (;;) { // oxlint-disable-next-line no-await-in-loop -- the generator is drained in order; its return value is the v2 state const n = await it.next(); if (n.done === true) return { state: n.value, pack: held.ok ? packCollectEnd(c) : held }; // Stryker disable next-line ConditionalExpression: a refused scan answers every later feed with the same refusal if (held.ok) held = packCollectFeed(c, n.value); } } // Where a commit want's bytes go: the first bytes are held until the pack header says how many objects there are (one: // the single-object parser, `feed`; more: a tag and the commit it peels to, collected). type CommitRoute = | { readonly tag: 'head'; readonly head: Uint8Array } | { readonly tag: 'single' } | { readonly tag: 'multi'; readonly c: PackCollector; readonly head: Uint8Array }; // A commit want's answer, read as it arrives (#172). Ok(null): the single-object parser has the pack (the caller checks // its trailer and object); otherwise the commit, or the refusal, held until the response was read to its end. async function commitWire(t: Transport, wire: PackFetch, feed: (chunk: Uint8Array) => Result, want: { readonly oid: Oid; readonly span: Span; readonly bust: boolean }): Promise> { // one record the closures below move along (a `let` the compiler would narrow to its first value) const at: { route: CommitRoute } = { route: { tag: 'head', head: new Uint8Array(0) } }; // the header and the first object-header byte: enough for the single-object parser to refuse a count it does not take const enough = PACK_HEADER_BYTES + 1; const open = (bytes: Uint8Array): Result => { const count = declaredPackCount(bytes); if (count === null || count === 1) { at.route = { tag: 'single' }; return feed(bytes); } const caps = packCaps({ limits: t.limits }); if (!caps.ok) return caps; const c = packCollectOpen(caps.value, true); // the caps passed packCaps, so the scan opens (packScanOpen) // Stryker disable next-line ConditionalExpression,BlockStatement: see above if (!c.ok) return c; at.route = { tag: 'multi', c: c.value, head: bytes.subarray(0, enough) }; return packCollectFeed(c.value, bytes); }; const step = (chunk: Uint8Array): Result => { switch (at.route.tag) { case 'single': return feed(chunk); case 'multi': return packCollectFeed(at.route.c, chunk); case 'head': { const head = concat([at.route.head, chunk]); // Stryker disable next-line ConditionalExpression,EqualityOperator: deciding on more bytes reads the same pack (the header is its first 12) if (head.length < enough) { at.route = { tag: 'head', head }; return UNIT; } return open(head); } } // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check return assertNever(at.route); }; let held: Result = UNIT; for await (const chunk of wire) if (held.ok) held = step(chunk); if (!held.ok) return held; // a response shorter than `enough`: decided on what came if (at.route.tag === 'head') { const opened = open(at.route.head); if (!opened.ok) return opened; } const route = at.route; if (route.tag !== 'multi') return ok(null); const parsed = packCollectEnd(route.c); if (!parsed.ok) return parsed; const found = commitFromPack(t, want.oid, parsed.value, want.span, want.bust); if (!found.ok || found.value !== null) return found; // the pack does not hold the commit: the single-object parser's refusal of its count const fed = feed(route.head); return fed.ok ? ok(null) : fed; } // The single object's member, inflated as its zlib bytes arrive (issue #172): the object is written into one // Uint8Array of its declared size and the zlib bytes are dropped as they are read, so the peak is the object plus the // inflate window, one chunk and one frame (streambound.ts), not the member as well. A refusal is held, not returned, until // the pack has been read to its trailer: spStep's and spFinish's refusals come first, as they did when the member was // inflated whole after spFinish. type Member = | { readonly tag: 'header' } | { readonly tag: 'open'; readonly stream: InflateStream; readonly out: Uint8Array; readonly at: number } | { readonly tag: 'done'; readonly out: Uint8Array; readonly consumed: number; readonly stats: InflateStats; readonly after: number } | { readonly tag: 'failed'; readonly error: InflateError }; const MEMBER_HEADER: Member = { tag: 'header' }; // Pull until the member wants more input, or ends, or is refused. function memberDrain(m: Member, end: boolean): Member { if (m.tag !== 'open') return m; let at = m.at; for (;;) { const r = end ? inflateEnd(m.stream) : inflatePull(m.stream); if (!r.ok) return { tag: 'failed', error: r.error }; const piece = r.value; switch (piece.kind) { case 'need': return { ...m, at }; // a declared bound: the stream never yields past the declared size, which is out's length case 'chunk': m.out.set(piece.bytes, at); at += piece.bytes.length; continue; case 'end': return { tag: 'done', out: m.out, consumed: piece.consumed, stats: piece.stats, after: inflateUnread(m.stream).length }; } // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check return assertNever(piece); } } function memberTake(m: Member, z: Uint8Array): Member { switch (m.tag) { case 'open': inflateFeed(m.stream, z); return memberDrain(m, false); case 'done': return { ...m, after: m.after + z.length }; // memberFeed opens a member before it hands it bytes, so 'header' does not reach here; a failed member drops them // Stryker disable next-line StringLiteral: 'header' is unreachable here, see above case 'header': case 'failed': return m; } // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check return assertNever(m); } // The zlib bytes spStep passed on. The member opens once the object header is read (spStep held its size to maxObject, // spInit's cap); bytes after the member's end are counted, not kept. function memberFeed(m: Member, sp: SpState, zlib: readonly Uint8Array[], maxObject: number): Member { let cur = m; if (cur.tag === 'header') { if (sp.phase !== 'body') return cur; const opened = inflateInit({ bound: declaredBound(sp.size, maxObject), wrap: zlibWrap(true) }); // the size is within maxObject (spStep), which parseLimits checked: the record always opens // Stryker disable next-line all: see above if (!opened.ok) return { tag: 'failed', error: opened.error }; cur = { tag: 'open', stream: opened.value, out: new Uint8Array(sp.size), at: 0 }; } for (const z of zlib) cur = memberTake(cur, z); return cur; } type Inflated = { readonly out: Uint8Array; readonly stats: InflateStats }; // The member once the pack has been read: its object, or the refusal it held. A member that ended before its zlib // bytes did is the trailing-bytes refusal. function memberFinish(m: Member): Result { const last = memberDrain(m, true); switch (last.tag) { // spFinish passed (so the object header was read, its size within maxObject): the member opened. inflateEnd // ends a member or refuses it, so it is not still open. // Stryker disable next-line all: unreachable, see above case 'header': return fail('internal', 'member never opened'); // Stryker disable next-line all: unreachable, see above case 'open': return fail('internal', 'member open after inflateEnd'); case 'failed': return err(last.error); case 'done': if (last.after !== 0) return fail('pack_corrupt', `pack: ${last.after} trailing bytes after object`); return ok({ out: last.out, stats: last.stats }); } // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check return assertNever(last); } // The single object, inflated: hashed against the want, its type checked. type Received = { readonly sp: SpFinished; readonly member: Member; readonly want: Single; readonly span: Span; readonly bust: boolean }; function singleObject(t: Transport, { sp, member, want, span, bust }: Received): Result { const { oid, expectType } = want; const inflated = memberFinish(member); if (!inflated.ok) return inflated; const { out, stats } = inflated.value; const got = hashObject(typeText(sp.type), out); span.set({ ...spAttrs(sp), 'git.pack.inflated.bytes': out.length, 'git.pack.inflate.blocks': stats.blocks, 'git.pack.inflate.stored': stats.stored, 'git.pack.inflate.fixed': stats.fixed, 'git.pack.inflate.dynamic': stats.dynamic, 'git.object.oid': got, 'git.object.type': typeText(sp.type) }); if (got !== oid) return fail('pack_corrupt', `object ${got} served for want ${oid}`); const checked = assertFetched(typeText(sp.type), out, t.limits); if (!checked.ok) return checked; const refused = refuseNonCommit(expectType, sp.type, oid); if (!refused.ok) return refused; if (expectType !== null && sp.type !== expectType) return fail('pack_corrupt', `expected ${typeText(expectType)}, got ${typeText(sp.type)}`); const tree = mapOption(checked.value, (entries): CheckedTree => ({ entries, maxTreeEntries: t.limits.maxTreeEntries })); return ok({ type: sp.type, data: out, oid, size: out.length, bust, tree }); } // oxlint-disable-next-line typescript/promise-function-async -- onceMore's promise, passed through function fetchSingle(t: Transport, want: Single): Promise> { const { oid, filter, deepen, expectType } = want; // The span is where the transport says whether the response came from a cache (fetchExchange, onceMore). // oxlint-disable-next-line typescript/promise-function-async -- the exchange's promise, passed through const once = (bust: boolean): Promise> => fetchExchange(t, { 'git.pack.cache.bust': bust }, async (span: Span): Promise> => { const maxObject = t.limits.maxObjectSize; // Pack bytes are held to maxPackBytes by v2Step (inside packStream) before a chunk reaches here; the zlib bytes are // bounded by spStep at zlibBound(sp.size) and inflated as they arrive (memberFeed), sp.size by maxObjectSize where // the header is read (readObjHeader, through spStep). let sp = spInit(maxObject); let member = MEMBER_HEADER; const feed = (chunk: Uint8Array): Result => { const r = spStep(sp, chunk); if (!r.ok) return r; sp = r.value.state; member = memberFeed(member, sp, r.value.zlib, maxObject); return UNIT; }; const wire = packStream(t, { wants: [oid], filter, deepen, bypassCache: bust }, span); // A commit id may arrive as the tag object plus the commit it peels to (commitWire). Other wants stream, and stop at // the header whose size is past maxObject (spStep refuses it) before the rest of the pack is read. if (expectType === TYPE.COMMIT) { const found = await commitWire(t, wire, feed, { oid, span, bust }); if (!found.ok) return found; if (found.value !== null) return ok(found.value); } else { for await (const chunk of wire) { const fed = feed(chunk); if (!fed.ok) return fed; } } const finished = spFinish(sp); if (!finished.ok) return finished; return singleObject(t, { sp: finished.value, member, want, span, bust }); }); return onceMore(once); } // The wide form of a filtered single-object fetch (see fetchObject): blob:none for a host that refuses tree:0, no filter // ('') for one that does not take `filter` at all. Bounded by limits.maxPackBytes (v2Step) and the pack parser's limits. export type WideFilter = 'blob:none' | ''; // The filter a wide fetch sent, as spans and messages name it. export const packName = (filter: string): string => (filter === '' ? 'unfiltered' : filter); export const fallbackName = (filter: WideFilter): string => (filter === '' ? 'none' : filter); // A wide fetch: the want, and the filter sent (a fallback's WideFilter, or a path walk's combine filter, #218). type Wide = { readonly oid: Oid; readonly deepen: number; readonly expectType: number | null; readonly filter: string }; // What a wide fetch left out as it read the pack: full blobs (a commit or tree want is neither, and a harvest keeps // commits and trees) and the deltas on them, by offset, with their declared bytes. type Dropped = { readonly offsets: Set; bytes: number }; // The objects a wide pack keeps (#172): all of them, unless the want cannot be a blob. Then a full blob is dropped as its // header arrives, and so is an OFS_DELTA on a dropped object (its base, and so its result, is a blob). A REF_DELTA is // kept; one whose base was dropped stays unresolved (wideObject). function wideKeep(want: Wide, dropped: Dropped): (h: ScannedHeader) => boolean { const dropBlobs = want.expectType === TYPE.COMMIT || want.expectType === TYPE.TREE; return (h: ScannedHeader): boolean => { // a header typed blob is a full object (a delta's type is OFS_DELTA or REF_DELTA); only an OFS base has an offset // Stryker disable next-line ConditionalExpression,LogicalOperator: a base that is not OFS has no offset, which no dropped offset equals const blob = h.type === TYPE.BLOB || (h.base.tag === 'ofs' && dropped.offsets.has(h.base.offset)); if (!dropBlobs || !blob) return true; dropped.offsets.add(h.offset); dropped.bytes += h.size; return false; }; } // The pack resolved: with nothing dropped, every base must be in the pack (resolvePack); with blobs dropped, what waits // on a base outside it is counted, not refused. function wideResolve(t: Transport, parsed: ParsedPack, dropped: Dropped): Result<{ readonly objects: Map; readonly unresolved: number }, PlyError> { if (dropped.offsets.size === 0) { // Stryker disable next-line ArrowFunction: null and a bare return both mean the base is not in the pack const r = resolvePack(parsed, () => null, t.limits); return r.ok ? ok({ objects: r.value.objects, unresolved: 0 }) : r; } // Stryker disable next-line ArrowFunction: null and a bare return both mean the base is not in the pack const r = resolvePackCollect(parsed, () => null, t.limits); return r.ok ? ok({ objects: r.value.objects, unresolved: r.value.unresolved.length }) : r; } // The wanted object out of the wide pack, its extras checked (checkWide) and harvested. When blobs were dropped, a delta // the resolver cannot resolve waits on a dropped base (a REF_DELTA on a blob, or on a delta of one) and is counted with // them: this fetch asked for no thin pack, so a base outside the pack is a blob it left out. type WideRead = { readonly parsed: ParsedPack; readonly dropped: Dropped }; function wideObject(t: Transport, { parsed, dropped }: WideRead, want: Wide, span: Span, bust: boolean): Result { const { oid, expectType } = want; const maxObject = t.limits.maxObjectSize; const resolved = wideResolve(t, parsed, dropped); if (!resolved.ok) return resolved; const { objects, unresolved } = resolved.value; const checked = checkWide(objects, k => k === oid, t.limits); if (!checked.ok) return checked; const skipped = checked.value; const found = mapGet(objects, oid); const left = dropped.offsets.size + unresolved; if (found.tag === 'none') return fail('pack_corrupt', `object ${oid} not in the ${packName(want.filter)} pack (${objects.size + left} objects)`); const wanted = found.value; if (wanted.data.length > maxObject) return err(limitExceeded('object size', wanted.data.length, maxObject)); const refused = refuseNonCommit(expectType, wanted.type, oid); if (!refused.ok) return refused; if (expectType !== null && wanted.type !== expectType) return fail('pack_corrupt', `expected ${typeText(expectType)}, got ${typeText(wanted.type)}`); let blobs = left, extraBytes = skipped.reduce((n, x) => n + x.bytes, dropped.bytes); for (const [k, o] of objects) { if (k !== oid) extraBytes += o.data.length; if (o.type === TYPE.BLOB) blobs++; } const harvested = harvest(t, objects, oid); if (!harvested.ok) return harvested; const h = harvested.value; const all = objects.size + skipped.length + left; span.set({ 'git.pack.objects': parsed.stats.count, 'git.pack.bytes': parsed.bytes, 'git.object.oid': oid, 'git.object.type': typeText(wanted.type), 'git.fetch.wide.objects': all, 'git.fetch.wide.extra_objects': all - 1, 'git.fetch.wide.extra_bytes': extraBytes, 'git.fetch.wide.blobs': blobs, 'git.fetch.wide.harvested': h.kept, 'git.fetch.wide.harvested_bytes': h.bytes, ...skippedAttrs(skipped) }); return ok({ type: wanted.type, data: wanted.data, oid, size: wanted.data.length, bust, tree: none, wide: true }); } // The wide pack read and its wanted object taken (wideObject), under one fetch span carrying `attrs`. // oxlint-disable-next-line typescript/promise-function-async -- onceMore's promise, passed through function wideFetch(t: Transport, want: Wide, attrs: Attrs): Promise> { const { oid, deepen, filter } = want; // oxlint-disable-next-line typescript/promise-function-async -- the exchange's promise, passed through const once = (bust: boolean): Promise> => fetchExchange(t, { 'git.pack.cache.bust': bust, ...attrs }, async (span: Span): Promise> => { const caps = packCaps({ limits: t.limits }); if (!caps.ok) return caps; // the pack is read through the scan as it arrives: its bytes and every zlib member are dropped once inflated, and // what is held is the objects wideKeep keeps (#172) const dropped: Dropped = { offsets: new Set(), bytes: 0 }; const c = packCollectOpen(caps.value, true, wideKeep(want, dropped)); // the caps passed packCaps, so the scan opens (packScanOpen) // Stryker disable next-line ConditionalExpression,BlockStatement: see above if (!c.ok) return c; const read = await collectWire(packStream(t, { wants: [oid], filter, deepen, bypassCache: bust }, span), c.value); if (!read.pack.ok) return read.pack; return wideObject(t, { parsed: read.pack.value, dropped }, want, span, bust); }); return onceMore(once); } // The fallback (fetchObject): what was asked (tree:0, or the filter the caller gave), what was sent, and whether this // request is the one that learned the refusal. type Fallback = { readonly oid: Oid; readonly deepen: number; readonly expectType: number | null; readonly learned: boolean; readonly filter: WideFilter; readonly requested: string }; // oxlint-disable-next-line typescript/promise-function-async -- wideFetch's promise, passed through const fetchWide = (t: Transport, { learned, requested, ...want }: Fallback): Promise> => wideFetch(t, want, { 'git.fetch.filter.fallback': fallbackName(want.filter), 'git.fetch.filter.requested': requested, 'git.fetch.filter.learned': learned }); // A commit and the trees `filter` brings with it (deepen 1), read as a wide pack is: every object keyed by its computed // hash, each commit and tree held to the caller's limits (checkWide, the commit the wanted object: an extra over a limit // is dropped, a malformed one fails the fetch), the trees harvested into the transport's store for the walk's tree:0 // reads (objectAt answers them locally). A path walk's first levels in one round trip (pathtrees.ts, #218). // oxlint-disable-next-line typescript/promise-function-async -- wideFetch's promise, passed through export const commitWithTrees = (t: Transport, oid: Oid, filter: string): Promise> => wideFetch(t, { oid, deepen: 1, expectType: TYPE.COMMIT, filter }, { 'git.fetch.path_trees': filter }); // The caller's oid, parsed once: a JS caller may pass anything, so it is read as String() renders it (a missing one // renders as no id) and quoted the same way. Not 40 lowercase hex is `fetchObject: oid must be 40-hex` (bad_request, no // `what`, as always). export function objectId(oid: unknown): Result { const text = String(oid); const id = parseOid(text); return id.ok ? id : fail('bad_request', `fetchObject: oid must be 40-hex, got ${JSON.stringify(text.slice(0, OID_PREVIEW))}`); } // fetchObject of a parsed id: ply's own callers, which hold an Oid, start here. // oxlint-disable-next-line typescript/promise-function-async -- probing's promise (or the harvest's answer), passed through export function objectAt(t: Transport, oid: Oid, options: Input = {}): Promise> { const { filter, deepen: asked, expectType } = withDefaults(options, objectWantDefaults()); // The depth to send (ObjectWantConfig): the caller's, else 1 when the want may be a commit and the server is not known // to lack `shallow`. Read at each send: the advertisement may be read between two of them. const mayBeCommit = expectType !== TYPE.TREE && expectType !== TYPE.BLOB; const depth = (): number => { if (asked !== null) return asked; return mayBeCommit && takesShallow(t.quirks) ? 1 : 0; }; // run(depth). When it fails carrying a depth this call added, an empty response, an ERR line or HTTP 400 (the shapes a // server refuses an argument in) sends ply to the advertisement (as for a refused filter: filterUnadvertised learns // `shallow` from the same line), and a server that lists no `shallow` is asked once more with no `deepen`, as before // the default: the answer is then the history, held to the pack limits. `shallow` listed: the first error stands. const ask = async (run: (deepen: number) => Promise>): Promise> => { const sent = depth(); const first = await run(sent); // once per quirks record: an advertisement already read for it has said whether `shallow` is listed if (first.ok || asked !== null || sent === 0 || shallowLearned(t.quirks)) return first; await filterUnadvertised(t, first.error, e => isEmptyResponse(e) || isArgumentRefusal(e)); return depth() === sent ? first : run(depth()); }; // a depth this call added: the caller gave none and one was sent const added = (sent: number): boolean => asked === null && sent !== 0; const narrow = filter === 'tree:0'; const held = narrow ? mapGet(t.objects, oid) : none; if (held.tag === 'some' && (expectType === null || held.value.type === expectType)) return Promise.resolve(ok({ type: held.value.type, data: held.value.data, oid, size: held.value.data.length, bust: false, tree: none, harvested: true })); // oxlint-disable-next-line typescript/promise-function-async -- ask's promise, passed through const wide = (sent: WideFilter, learned: boolean): Promise> => ask(deepen => fetchWide(t, { oid, deepen, expectType, learned, filter: sent, requested: filter })); // oxlint-disable-next-line typescript/promise-function-async -- fetchSingle's promise, passed through if (filter === '') return ask(deepen => fetchSingle(t, { oid, filter, deepen, expectType })); // one probe at a time per quirks record (transport.ts probing): the checks below read it after any wait return probing(t.quirks, narrow ? 'tree:0' : 'filter', async (): Promise> => { const harvested = narrow ? mapGet(t.objects, oid) : none; if (harvested.tag === 'some' && (expectType === null || harvested.value.type === expectType)) return ok({ type: harvested.value.type, data: harvested.value.data, oid, size: harvested.value.data.length, bust: false, tree: none, harvested: true }); if (t.quirks.treeFilter === 'none') return wide('', false); if (narrow && t.quirks.treeFilter === 'blob:none') return wide('blob:none', false); const sent = depth(); const single = await fetchSingle(t, { oid, filter, deepen: sent, expectType }); if (single.ok) return single; if (narrow && isFilterRejection(single.error)) { t.quirks.treeFilter = 'blob:none'; return wide('blob:none', true); } if (await filterUnadvertised(t, single.error)) return wide('', true); // an ERR line or HTTP 400 that answered a depth this call added: the advertisement is read for `shallow` (see `ask`; // not read above, where an unlisted `filter` would change the request main sends) const unlisted = added(sent) && !shallowLearned(t.quirks) && (await filterUnadvertised(t, single.error, isArgumentRefusal)); // the advertisement was just read: a depth this call added that the server does not take is dropped (see `ask`). // The request without it is main's, and its answer is handled as main handles it: no `filter` listed is the // unfiltered pack, a refused tree:0 is blob:none, an empty response is the advertisement's to explain. if (depth() === sent) return single; if (unlisted) return wide('', true); const again = await fetchSingle(t, { oid, filter, deepen: depth(), expectType }); if (again.ok) return again; if (narrow && isFilterRejection(again.error)) { t.quirks.treeFilter = 'blob:none'; return wide('blob:none', true); } return (await filterUnadvertised(t, again.error)) ? wide('', true) : again; }); } // One object by oid: a bad oid, a fetch failure, a verification failure are the Err. // oxlint-disable-next-line typescript/promise-function-async -- objectAt's promise, passed through export function fetchObject(t: Transport, want: ObjectWant): Promise> { const id = objectId(want.oid); return id.ok ? objectAt(t, id.value, want) : Promise.resolve(id); }