main
1// Pack subset (#235): a new, valid pack of chosen entries of an existing one, read through a range reader, each entry's
2// compressed bytes copied unchanged (nothing inflated or deflated). How a push too large for a host's cap is cut into
3// several, or part of a received pack relayed, in memory bounded by a chunk whatever the pack's size.
4//
5// The selection is a list of the source's entry indices (from scanPackIndex, #236) in the order they are to be written,
6// which may differ from the source's. It is copied at the call (4 bytes an entry): the array is the caller's to reuse. Each entry gets a rewritten header: the same type and size, an OFS_DELTA's
7// distance recomputed for its new position, a REF_DELTA's base id unchanged (Cloudflare Artifacts packs are REF only, so
8// they pass through as they are). An OFS base must be written earlier in the new pack; one that is not (left out, or
9// later in the order) is refused by default, or written as a REF_DELTA to the base's id with `outsideBase: 'ref'` when
10// that id is known: the scan index knows a whole object's (oids on); `resolved` (resolvePackRange over the same pack)
11// knows every entry's, so with it a chain is cut anywhere, below a delta base too. A REF base outside the selection
12// stays a REF: such a pack is thin, and the receiver supplies the base (receive-pack's index-pack --fix-thin) or
13// refuses.
14//
15// One decision, shared: `entryWrite` and `baseId` say how an entry is written given where its base is, and the planner
16// (packplan.ts) plans with the same two functions, so a batch it plans is one this module builds from the same
17// `resolved`.
18//
19// Base first. With `resolved`, a selected base is written before the first selected entry resting on it (through any
20// chain), whatever the selection's order; entries with no such tie keep the caller's order. Stock git refuses a thin
21// pack in which a REF_DELTA precedes its base when that base is itself a delta the receiver already holds (it appends
22// the base from its store, then meets it in the pack: "appears twice in the pack", measured against git 2.50
23// index-pack --strict --fix-thin). Without `resolved` a REF_DELTA's delta base cannot be told from a base outside the
24// pack, so the order is the caller's to get right (the planner's batches are base first).
25//
26// `extras` are small objects ply encodes itself (a commit, a generated tree), deflated before anything is streamed and
27// written after the selected entries. Their base may be none or a REF; an OFS base inside extras is refused, as is any
28// extra a pack cannot hold (a type that is not commit, tree, blob or tag; a REF base that is not a 40-hex id).
29//
30// `length` is exact before the first byte, so the stream feeds pushPack (#234) with a known length. Consecutive source
31// entries are read with one range read; the SHA-1 trailer is computed while streaming. Held: per selected entry 8 bytes
32// (its new offset) + 4 (its OFS base's position) + 4 (its index, the copy) + 1 (its kind), per source entry 4 + 1
33// (position, selected mark), per entry rewritten as a REF_DELTA its base's 20-byte id, the extras' deflated bytes, and
34// the chunk in flight; with `resolved`, also 4 bytes per selected entry for the base-first order (the copy is reordered
35// into a second Uint32Array) and 1 per source entry while ordering. A source that serves fewer or more bytes than a range asks for
36// errors the stream (pack_corrupt) before the trailer, so the receiver never gets a whole pack from a short read.
37//
38// The source is the caller's storage and may never answer. Cancelling the stream settles at once all the same: no await
39// on the source (opening a range, reading it, cancelling it) is made without `Halt.race` (rangesource.ts), the range in flight is
40// cancelled and the source's abort signal fired without waiting for either, and whatever a read answers later is
41// dropped (a range that opens late is cancelled).
42import type { PlyError } from './errors.js';
43import { fail, plyError, argsError } from './errors.js';
44import type { Result } from './result.js';
45import { ok } from './result.js';
46import { withDefaults, fromNullish } from './absent.js';
47import type { Input } from './absent.js';
48import type { RangeSource, Halt } from './rangesource.js';
49import { HALTED, haltOn, drop, cancelRange, cancelLate } from './rangesource.js';
50import type { ScannedPack } from './packscanindex.js';
51import type { ResolvedIndex } from './packrangeresolve.js';
52import type { PackInput } from './pack.js';
53import { encodeObjHeader, encodeOfsDistance, entryHead, inputBase, packHeader, zlibDeflate } from './pack.js';
54import { streamingSha1, fromHex } from './sha1.js';
55import { concat } from './bytes.js';
56import { OBJ_COMMIT, OBJ_TAG, OBJ_OFS_DELTA, OBJ_REF_DELTA, PACK_HEADER_BYTES, PACK_TRAILER_BYTES } from './format.js';
58// In-range indexed reads of columns this module sized or the index sized; noUncheckedIndexedAccess cannot see that.
59/* oxlint-disable typescript/no-unsafe-type-assertion, typescript/non-nullable-type-assertion-style -- in-range indexed reads, see above */
61export type OutsideBase = 'refuse' | 'ref';
62export type PackSubsetOpts = {
63 readonly extras: readonly PackInput[];
64 readonly outsideBase: OutsideBase;
65 // every entry's id (resolvePackRange over this pack), or null: only a whole object's id is known (the scan index)
66 readonly resolved: ResolvedIndex | null;
67};
68export type PackSubset = { readonly stream: ReadableStream<Uint8Array>; readonly length: number; readonly count: number };
69// A fresh record per call: withDefaults fills it in place.
70const defaultOpts = (): PackSubsetOpts => ({ extras: [], outsideBase: 'refuse', resolved: null });
72// A pack's object count is a 32-bit field.
73const MAX_COUNT = 0xff_ff_ff_ff;
74// How an entry is written: as stored (a whole object, or a REF_DELTA with its base id), as an OFS_DELTA to its base's
75// new position, as a REF_DELTA in place of the stored OFS_DELTA, or not at all.
76export const WRITE_WHOLE = 0;
77export const WRITE_OFS = 1;
78export const WRITE_REF = 2;
79export const WRITE_REF_FROM_OFS = 3;
80export const WRITE_REFUSED = 4;
81export type EntryWrite = typeof WRITE_WHOLE | typeof WRITE_OFS | typeof WRITE_REF | typeof WRITE_REF_FROM_OFS | typeof WRITE_REFUSED;
82const KIND_WHOLE = WRITE_WHOLE;
83const KIND_OFS = WRITE_OFS;
84const KIND_REF = WRITE_REF;
85const KIND_REF_FROM_OFS = WRITE_REF_FROM_OFS;
86const NOT_WRITTEN = -1;
88// The one rule for writing an entry whose stored type is `stored`: `baseWritten` says its OFS base is earlier in the
89// same new pack, `baseIdKnown` that `baseId` names that base. The planner and the builder both decide with this.
90export function entryWrite(stored: number, baseWritten: boolean, baseIdKnown: boolean, outsideBase: OutsideBase): EntryWrite {
91 if (stored === OBJ_REF_DELTA) return WRITE_REF;
92 if (stored !== OBJ_OFS_DELTA) return WRITE_WHOLE;
93 if (baseWritten) return WRITE_OFS;
94 return outsideBase === 'ref' && baseIdKnown ? WRITE_REF_FROM_OFS : WRITE_REFUSED;
95}
96// Bytes an OFS_DELTA header spends on a distance (the planner counts written headers with it).
97export const ofsDistanceBytes = (distance: number): number => encodeOfsDistance(distance).length;
99type Plan = {
100 readonly pack: ScannedPack;
101 // the plan's own copy of the caller's selection, taken at the call, in write order: the offsets below are planned
102 // from these values, so the stream must write from the same ones whatever the caller does with its array afterwards
103 readonly selection: Uint32Array;
104 readonly resolved: ResolvedIndex | null;
105 readonly kind: Uint8Array;
106 readonly basePos: Int32Array;
107 readonly newOffset: Float64Array;
108 // by position, the base id of each entry rewritten as a REF_DELTA: the answer plan() checked, asked for once
109 readonly baseIds: Map<number, Uint8Array>;
110 readonly extras: ReadonlyArray<{ readonly head: Uint8Array; readonly z: Uint8Array }>;
111 readonly body: number;
112};
114const cancelled = (): PlyError => plyError('aborted', 'pack subset: the stream was cancelled');
116const refuse = (what: string, message: string): Result<never, PlyError> => fail('bad_request', `pack subset: ${message}`, { what });
118// The id of whole object `i`, if the index computed it.
119function wholeOid(pack: ScannedPack, i: number): string | null {
120 const e = pack.entry(i);
121 return e.tag === 'some' ? e.value.oid : null;
122}
123// An object id as a REF_DELTA header carries it: 40 lowercase hex, what parseOid (oid.ts) accepts. Spelled here because
124// this file is at its import budget.
125const OID_TEXT = /^[0-9a-f]{40}$/u;
126// What a caller's `resolved.oid()` threw: carried to the refusal as its cause, never swallowed and never let out.
127export type Threw = { readonly threw: unknown };
128// A caller's `resolved.oid()` that returned nothing at all (no Option): refused in those words, with no cause.
129export type Nothing = { readonly nothing: true };
130const NOTHING: Nothing = { nothing: true };
131// The id `resolved` gives entry `b`, checked here, where it is used. A resolved index is the caller's object (made by
132// hand, or read back from storage): an `oid` that is missing or answers anything but 40 lowercase hex names no base
133// (null), and one that throws is reported with what it threw. Either way the entry is refused (by the planner and the
134// builder alike) and nothing but an object id is ever written into a REF_DELTA header. Asked only for a base outside
135// the batch, so the check costs nothing elsewhere.
136function resolvedId(resolved: ResolvedIndex, b: number): string | Threw | Nothing | null {
137 try {
138 const answer = fromNullish(resolved.oid(b));
139 if (answer.tag === 'none') return NOTHING;
140 const id = answer.value;
141 if (id.tag !== 'some') return null;
142 const value: unknown = id.value;
143 return typeof value === 'string' && OID_TEXT.test(value) ? value : null;
144 } catch (threw) {
145 return { threw };
146 }
147}
148// The id of entry `b` as a REF_DELTA would name it: from `resolved` when given (any entry), else the scan index's (a
149// whole object, oids on); null when neither knows it, and what `resolved.oid()` threw when it did.
150export function baseId(pack: ScannedPack, resolved: ResolvedIndex | null, b: number): string | Threw | Nothing | null {
151 return resolved === null ? wholeOid(pack, b) : resolvedId(resolved, b);
152}
153// The REF base id entry `i` names.
154function refOid(pack: ScannedPack, i: number): string | null {
155 const e = pack.entry(i);
156 return e.tag === 'some' && e.value.base.tag === 'ref' ? e.value.base.oid : null;
157}
159// Entry k's header in the new pack.
160function headerOf(p: Plan, k: number): Uint8Array {
161 const i = p.selection[k] as number;
162 const c = p.pack.columns;
163 const size = c.size[i] as number;
164 switch (p.kind[k] as number) {
165 case KIND_OFS: return concat([encodeObjHeader(OBJ_OFS_DELTA, size), encodeOfsDistance((p.newOffset[k] as number) - (p.newOffset[p.basePos[k] as number] as number))]);
166 case KIND_REF: return concat([encodeObjHeader(OBJ_REF_DELTA, size), fromHex(refOid(p.pack, i) as string)]);
167 case KIND_REF_FROM_OFS: return concat([encodeObjHeader(OBJ_REF_DELTA, size), p.baseIds.get(k) as Uint8Array]);
168 default: return encodeObjHeader(c.storedType[i] as number, size);
169 }
170}
172// The copy with every selected base moved before the first entry that rests on it; otherwise the caller's order.
173// `queued` marks each entry once, so a base column that loops (a damaged index) still ends: at most one step an entry,
174// and at most `copy.length` entries written. A chain is written into `out` as it is walked, then turned round in place.
175function baseFirst(copy: Uint32Array, selected: Uint8Array, resolved: ResolvedIndex): Uint32Array {
176 const base = resolved.columns.base;
177 const queued = new Uint8Array(selected.length);
178 const out = new Uint32Array(copy.length);
179 let w = 0;
180 for (let k = 0; k < copy.length; k++) {
181 const from = w;
182 let u = copy[k] as number;
183 while (u >= 0 && u < selected.length && selected[u] === 1 && queued[u] !== 1) {
184 queued[u] = 1;
185 out[w++] = u;
186 u = base[u] as number;
187 }
188 out.subarray(from, w).reverse();
189 }
190 return out;
191}
193// Extra j as a pack can hold it: bytes, and either a whole object of one of the four types (commit 1 .. tag 4; 0, 5 and
194// the delta types are not whole objects) or a REF_DELTA to a 40-hex id, whose own type is the delta's and is not read.
195// The values are a JavaScript caller's, so they are checked as unknown: buildPack's encoder checks none of this
196// (pack.ts), and an extra it would write wrong is a pack of the declared length that no git can parse.
197function checkExtra(j: number, given: PackInput): Result<null, PlyError> {
198 // not an object at all (null, a missing value, a number): refused before a property of it is read
199 const o: unknown = given;
200 if (typeof o !== 'object' || o === null) return refuse('pack subset extras', `extra ${j} is not an object`);
201 const data: unknown = given.data;
202 if (!(data instanceof Uint8Array)) return refuse('pack subset extras', `extra ${j} has data that is not a Uint8Array`);
203 const base = inputBase(given);
204 const tag: unknown = base.tag;
205 if (tag === 'ofs') return refuse('pack subset extras', `extra ${j} has an OFS base; give a REF base or none`);
206 if (tag === 'ref') {
207 const oid: unknown = base.tag === 'ref' ? base.oid : null;
208 return typeof oid === 'string' && OID_TEXT.test(oid) ? ok(null) : refuse('pack subset extras', `extra ${j} has a REF base that is not a 40-hex object id`);
209 }
210 if (tag !== 'none') return refuse('pack subset extras', `extra ${j} has a base that is neither none nor a REF`);
211 const type: unknown = given.type;
212 if (typeof type !== 'number' || !Number.isInteger(type) || type < OBJ_COMMIT || type > OBJ_TAG) return refuse('pack subset extras', `extra ${j} has type ${String(type)}; a whole object is a commit (1), tree (2), blob (3) or tag (4)`);
213 return ok(null);
214}
216// The entries to write, checked and copied (a caller's later change to its array changes nothing), in write order (base
217// first with `resolved`), with the marks the layout fills in. Every extra is checked first, before any is deflated.
218type Order = { readonly order: Uint32Array; readonly pos: Int32Array; readonly selected: Uint8Array };
219function ordered(pack: ScannedPack, selection: readonly number[], opts: PackSubsetOpts): Result<Order, PlyError> {
220 const n = selection.length;
221 const total = n + opts.extras.length;
222 if (total > MAX_COUNT) return refuse('pack subset count', `${total} objects is more than a pack can count`);
223 // every extra is checked before any is deflated, and before the selection is walked
224 for (const [j, o] of opts.extras.entries()) {
225 const checked = checkExtra(j, o);
226 if (!checked.ok) return checked;
227 }
228 // where each source entry is written (position in the new pack), NOT_WRITTEN until it is
229 const pos = new Int32Array(pack.count).fill(NOT_WRITTEN);
230 const selected = new Uint8Array(pack.count);
231 // distinct entries of the pack are at most its count, so the copy is sized by the pack, never by the caller's array
232 if (n > pack.count) return refuse('pack subset selection', `${n} entries selected from a pack of ${pack.count}: one is selected twice or is not in the pack`);
233 const copy = new Uint32Array(n);
234 for (let k = 0; k < n; k++) {
235 const i = selection[k] as number;
236 if (!Number.isInteger(i) || i < 0 || i >= pack.count) return refuse('pack subset selection', `entry ${String(i)} is not in the pack (0..${pack.count - 1})`);
237 if (selected[i] === 1) return refuse('pack subset selection', `entry ${i} is selected twice`);
238 selected[i] = 1;
239 copy[k] = i;
240 }
241 return ok({ order: opts.resolved === null ? copy : baseFirst(copy, selected, opts.resolved), pos, selected });
242}
244// Each entry's kind, header and offset in the new pack, in write order; then the extras.
245async function plan(pack: ScannedPack, selection: readonly number[], opts: PackSubsetOpts): Promise<Result<Plan, PlyError>> {
246 const checked = ordered(pack, selection, opts);
247 if (!checked.ok) return checked;
248 const { order, pos, selected } = checked.value;
249 const c = pack.columns;
250 const n = order.length;
251 const p: Plan = { pack, selection: order, resolved: opts.resolved, kind: new Uint8Array(n), basePos: new Int32Array(n).fill(NOT_WRITTEN), newOffset: new Float64Array(n), baseIds: new Map(), extras: [], body: 0 };
252 let offset = PACK_HEADER_BYTES;
253 for (let k = 0; k < n; k++) {
254 const i = order[k] as number;
255 const stored = c.storedType[i] as number;
256 if (stored === OBJ_OFS_DELTA) {
257 const b = c.baseIndex[i] as number;
258 const at = pos[b] as number;
259 // the id is asked for only when it would be written (the base is not earlier in this pack), and once: the
260 // answer checked here is the one the header carries, so a `resolved` that answers otherwise later changes nothing
261 const id = at === NOT_WRITTEN ? baseId(pack, opts.resolved, b) : null;
262 if (id !== null && typeof id !== 'string') {
263 return 'threw' in id
264 ? fail('bad_request', `pack subset: the resolved index's oid() threw for entry ${b}, the base of entry ${i}`, { what: 'pack subset base', cause: id.threw })
265 : refuse('pack subset base', `the resolved index's oid() returned nothing for entry ${b}, the base of entry ${i}`);
266 }
267 const write = entryWrite(stored, at !== NOT_WRITTEN, id !== null, opts.outsideBase);
268 if (write === WRITE_OFS) {
269 p.kind[k] = KIND_OFS;
270 p.basePos[k] = at;
271 } else if (write === WRITE_REF_FROM_OFS && id !== null) {
272 p.kind[k] = KIND_REF_FROM_OFS;
273 p.baseIds.set(k, fromHex(id));
274 } else {
275 const where = selected[b] === 1 ? 'is written after it' : 'is not selected';
276 const hint = opts.outsideBase === 'ref' ? ' and its id is unknown (a delta and no `resolved`, a `resolved` with no valid id for it, or oids off)' : '; outsideBase: \'ref\' writes it as a REF_DELTA when the base id is known';
277 return refuse('pack subset base', `entry ${i} is an OFS_DELTA whose base, entry ${b}, ${where}${hint}`);
278 }
279 } else if (stored === OBJ_REF_DELTA) {
280 if (refOid(pack, i) === null) return refuse('pack subset base', `entry ${i} is a REF_DELTA with no base id in the index`);
281 p.kind[k] = KIND_REF;
282 } else {
283 p.kind[k] = KIND_WHOLE;
284 }
285 p.newOffset[k] = offset;
286 pos[i] = k;
287 offset += headerOf(p, k).length + (c.compressed[i] as number);
288 }
289 const extras: Array<{ head: Uint8Array; z: Uint8Array }> = [];
290 for (const o of opts.extras) {
291 // the extras are small and few; in order, as the pack is
292 // oxlint-disable-next-line no-await-in-loop -- see above
293 const z = await zlibDeflate(o.data);
294 const head = entryHead(o, offset, []).head;
295 extras.push({ head, z });
296 offset += head.length + z.length;
297 }
298 return ok({ ...p, extras, body: offset });
299}
301// Exactly `length` bytes from a range reader, cut into the views the caller asks for (never a copy). `take(n)` yields
302// n bytes and returns null, or returns the refusal; `skip(n)` drops n; `end()` checks the range is used up. A refusal is
303// a value, never a throw (docs/STACK.md rule 1): the stream below errors itself with it.
304type Cut = {
305 readonly take: (want: number) => AsyncGenerator<Uint8Array, PlyError | null, unknown>;
306 readonly skip: (want: number) => Promise<PlyError | null>;
307 readonly end: () => Promise<PlyError | null>;
308};
309// `bare`: the reader is the platform's own (platformReader), so a chunk read is awaited as it is, with no promise and no
310// hop added per chunk: the halt's abort listener cancels the reader, which settles a read in flight (done), and the
311// check after the await sees the halt. Any other reader may never answer its cancel, so its reads go through halt.race.
312function cutter(reader: ReadableStreamDefaultReader<Uint8Array>, start: number, length: number, halt: Halt, bare: boolean): Cut {
313 let chunk: Uint8Array = new Uint8Array(0);
314 let at = 0;
315 let got = 0;
316 const long = (): PlyError => plyError('pack_corrupt', `pack subset: the range at ${start} served more than its ${length} bytes`);
317 // the next chunk of the range, or why there is none
318 async function refill(): Promise<PlyError | null> {
319 const r = bare ? await reader.read() : await halt.race(reader.read(), drop);
320 if (r === HALTED || halt.halted()) return cancelled();
321 if (r.done) return plyError('pack_corrupt', `pack subset: the range at ${start} ended after ${got} of ${length} bytes`);
322 if (got + r.value.length > length) return long();
323 chunk = r.value;
324 at = 0;
325 got += chunk.length;
326 return null;
327 }
328 async function* take(want: number): AsyncGenerator<Uint8Array, PlyError | null, unknown> {
329 let left = want;
330 while (left > 0) {
331 if (at >= chunk.length) {
332 // oxlint-disable-next-line no-await-in-loop -- one stream, read in order
333 const refused = await refill();
334 if (refused !== null) return refused;
335 continue;
336 }
337 const n = Math.min(left, chunk.length - at);
338 yield chunk.subarray(at, at + n);
339 at += n;
340 left -= n;
341 }
342 return null;
343 }
344 async function skip(want: number): Promise<PlyError | null> {
345 let left = want;
346 while (left > 0) {
347 if (at >= chunk.length) {
348 // oxlint-disable-next-line no-await-in-loop -- see above
349 const refused = await refill();
350 if (refused !== null) return refused;
351 continue;
352 }
353 const n = Math.min(left, chunk.length - at);
354 at += n;
355 left -= n;
356 }
357 return null;
358 }
359 async function end(): Promise<PlyError | null> {
360 if (at < chunk.length) return long();
361 const r = bare ? await reader.read() : await halt.race(reader.read(), drop);
362 if (r === HALTED || halt.halted()) return cancelled();
363 return r.done ? null : long();
364 }
365 return { take, skip, end };
366}
368// The platform's reader on a range whose getReader is the platform's own. null for any other range, which is then read
369// through its own reader: a class that defines its own getReader (a subclass that serves its bytes through it, with no
370// underlying source at all) and an object that is not a platform ReadableStream (the brand check throws).
371// oxlint-disable-next-line typescript/unbound-method -- only ever called with the range as its `this`, just below
372const getPlatformReader: (this: ReadableStream<Uint8Array>) => ReadableStreamDefaultReader<Uint8Array> = ReadableStream.prototype.getReader;
373function platformReader(range: ReadableStream<Uint8Array>): ReadableStreamDefaultReader<Uint8Array> | null {
374 // oxlint-disable-next-line typescript/unbound-method -- compared, never called
375 if (range.getReader !== getPlatformReader) return null;
376 try { return getPlatformReader.call(range); } catch { return null; }
377}
379// Runs of entries consecutive in the source, written consecutively: one range read each.
380function* runs(p: Plan): Generator<{ readonly from: number; readonly to: number }> {
381 const sel = p.selection;
382 let from = 0;
383 for (let k = 1; k <= sel.length; k++) {
384 if (k === sel.length || (sel[k] as number) !== (sel[k - 1] as number) + 1) {
385 yield { from, to: k };
386 from = k;
387 }
388 }
389}
391// One run's bytes: each entry's new header, then its compressed bytes from the range; null at the end, or the refusal.
392async function* run(p: Plan, cut: Cut, from: number, to: number): AsyncGenerator<Uint8Array, PlyError | null, unknown> {
393 const c = p.pack.columns;
394 for (let k = from; k < to; k++) {
395 const i = p.selection[k] as number;
396 // oxlint-disable-next-line no-await-in-loop -- one stream, read in order
397 const skipped = await cut.skip(c.headerBytes[i] as number);
398 if (skipped !== null) return skipped;
399 yield headerOf(p, k);
400 const took = yield* cut.take(c.compressed[i] as number);
401 if (took !== null) return took;
402 }
403 return cut.end();
404}
406// The new pack's bytes before its trailer; null when all were written, or the refusal that stopped them.
407async function* pieces(source: RangeSource, p: Plan, halt: Halt): AsyncGenerator<Uint8Array, PlyError | null, unknown> {
408 const c = p.pack.columns;
409 yield packHeader(p.selection.length + p.extras.length);
410 for (const r of runs(p)) {
411 if (r.to === r.from) continue;
412 const first = p.selection[r.from] as number;
413 const last = p.selection[r.to - 1] as number;
414 const start = c.offset[first] as number;
415 const length = (c.offset[last] as number) + (c.headerBytes[last] as number) + (c.compressed[last] as number) - start;
416 // oxlint-disable-next-line no-await-in-loop -- one run after another, in pack order
417 const opened = await halt.race(source.read(start, length, halt.signal), cancelLate);
418 if (opened === HALTED) return cancelled();
419 if (!opened.ok) return opened.error;
420 const own = platformReader(opened.value);
421 const reader = own === null ? opened.value.getReader() : own;
422 halt.reading(reader);
423 let outcome: PlyError | null | 'open' = 'open';
424 try {
425 outcome = yield* run(p, cutter(reader, start, length, halt, own !== null), r.from, r.to);
426 } finally {
427 halt.reading(null);
428 // a range not read to its end (a refusal, the consumer cancelling) is cancelled, not left open; the source is
429 // asked, not waited for
430 if (outcome !== null) cancelRange(reader, null);
431 reader.releaseLock();
432 }
433 if (outcome !== null) return outcome;
434 }
435 for (const x of p.extras) { yield x.head; yield x.z; }
436 return null;
437}
439// The selected entries of `pack`, read from `source`, as a new pack (see the top of this file). Refusals of the
440// selection, its bases or the extras are a Result before any byte is read; a range that fails or serves the wrong
441// length errors the stream before its trailer.
442export async function packSubset(source: RangeSource, pack: ScannedPack, selection: readonly number[], input: Input<PackSubsetOpts> = {}): Promise<Result<PackSubset, PlyError>> {
443 const shape = argsError('packSubset', [['source', source, 'object'], ['pack', pack, 'object'], ['selection', selection, 'array'], ['options', input, 'object']]);
444 if (shape !== null) return shape;
445 const opts = withDefaults(input, defaultOpts());
446 if (opts.resolved !== null && opts.resolved.count !== pack.count) return refuse('pack subset resolved', `the resolved index has ${opts.resolved.count} entries, the pack ${pack.count}`);
447 if (pack.bytes > source.size) return refuse('pack subset source', `the source holds ${source.size} bytes; the indexed pack is ${pack.bytes}`);
448 // a JavaScript caller's value, checked as unknown: null, a number or an array-like is refused before it is counted
449 const extras: unknown = opts.extras;
450 if (!Array.isArray(extras)) return refuse('pack subset extras', 'extras is not an array');
451 const planned = await plan(pack, selection, opts);
452 if (!planned.ok) return planned;
453 const p = planned.value;
454 const length = p.body + PACK_TRAILER_BYTES;
455 // the stream's own signal: aborted by its consumer's cancel, given to every source.read
456 const abort = new AbortController();
457 const halt = haltOn(abort.signal);
458 const it = pieces(source, p, halt);
459 const sha = streamingSha1();
460 let sent = 0;
461 const stream = new ReadableStream<Uint8Array>({
462 async pull(ctl): Promise<void> {
463 let r: IteratorResult<Uint8Array, PlyError | null>;
464 // a range reader whose stream errors (its own failure) rejects here
465 try { r = await it.next(); } catch (e) { if (!halt.halted()) ctl.error(e); return; }
466 // cancelled while this read was in flight: nothing more is enqueued, whatever the read came to
467 if (halt.halted()) return;
468 if (r.done === true) {
469 if (r.value !== null) { ctl.error(r.value); return; }
470 if (sent !== p.body) { ctl.error(plyError('internal', `pack subset: wrote ${sent} bytes, planned ${p.body}`)); return; }
471 ctl.enqueue(fromHex(await sha.hex()));
472 ctl.close();
473 return;
474 }
475 sent += r.value.length;
476 await sha.update(r.value);
477 if (halt.halted()) return;
478 ctl.enqueue(r.value);
479 },
480 // Settles at once, whatever the source does: the read in flight is released (Halt), and the generator unwinds
481 // behind it, or from the yield it is parked at, without being waited for.
482 cancel(reason: unknown): void {
483 abort.abort(reason);
484 it.return(null).catch(drop);
485 },
486 }, { highWaterMark: 0 });
487 return ok({ stream, length, count: p.selection.length + p.extras.length });
488}
Diagnostics4 git calls · 431 ms
CallStatusSizeTime0431 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2009.0 KiB425 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=425 git.response.bytes=9252 git.response.cache=MISS git.response.duration_ms=425 git.fetch.wants=113bb756ace2d13f8dc302d40273d2c37c0c6022 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=9252 git.pkt.payload.max=9231 git.sideband.band1.bytes=9230 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e303f8b8a27929
- Request
- 7537fef6…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format