GitHubivarvong

ply

main
ply6 git calls76 ms30 KiB4 cachedprotocol v2
647 lines32 KiBRaw
1// Every entry's oid and resolved type for a pack too large to hold (#248): what `git index-pack` computes, over a spooled
2// pack read back through a RangeSource, in memory bounded by object size rather than pack size. The layout comes from
3// scanPackIndex (#236); this applies the deltas that scan cannot.
4//
5// Why: a REF_DELTA names its base by oid, and a delta's oid is known only once it is applied. Cloudflare Artifacts sends
6// every delta as REF_DELTA, so on its packs any chain deeper than one is unknown to the scan, and children-first
7// planning (#237) needs every type.
8//
9// How: from each whole object, depth first over the deltas that rest on it (OFS children by base index, REF children by
10// the base's oid). A delta's bytes are read back through the source, inflated and applied to its base. Nothing holds a
11// chain. All the deltas resting on a base are applied while that base is held; of the results that something rests on,
12// the last is pinned (it is the next base, so a linear chain builds every object exactly once whatever its size) and
13// the others go to a small LRU (`cacheBytes`). A base in neither is rebuilt from its chain root on demand, each frame
14// holding nothing while its base is made, and every base made on the way is put in the LRU: a fork whose siblings
15// fell out of the cache walks the chain once per cache-full of siblings, not once per sibling.
16//
17// Memory: one base + one pinned result + one result + one delta's instructions + the LRU, at most
18// max(maxObjectSize, maxDeltaResult) + 3 x maxDeltaResult + cacheBytes, whatever the pack's size, depth or shape.
19//
20// Work (bytes inflated from the source + bytes of delta results produced + one per child a base's deltas are looked at
21// for) is bounded by what the pack declares, not by
22// its compressed size (delta-compressed history is legitimately hundreds of times its pack). The floor is every member
23// inflated once (the scan's sizes) plus every delta applied once (its header's result size, capped by maxDeltaResult).
24// A work budget allows LIMIT_RESOLVE_FACTOR x the floor reached so far (the factor is the room for rebuilds), never
25// less than maxInflatedTotal and never more than the ceiling, limits.maxResolveWork as parsed
26// (8 x maxInflatedTotal unless the caller sets it; parseLimits refuses it below maxInflatedTotal, #271): CPU is at most the factor times the declared sizes and at most the ceiling, which is what stops a delta bomb
27// (every first-time result raises the floor). A retain budget allows maxInflatedTotal, flat.
28//
29// Verified: each member's zlib checksum, length and declared size; every delta against its base's size; every oid
30// computed from the bytes (never taken from the pack); a base read back a second time rehashed against the oid it had
31// the first time (a spool that changed under us is pack_corrupt, not a wrong oid); a thin base hashed against the oid
32// that named it. Refused: a base in neither the pack nor `bases` (pack_corrupt, "not found"; a REF cycle cannot
33// resolve, so it ends there too), a chain deeper than maxDeltaChain, and work past the budget (limit_exceeded).
34//
35// Kept per entry in the result: RESOLVED_ENTRY_BYTES (oid 20 + type 1 + base 4 + depth 4 + size 8 + the oid-ordered
36// index 4).
37// Held per entry only while resolving: child links (4 per entry + 4 per OFS delta), each REF delta's base id and
38// position (24), and a known flag (1).
39import type { PlyError } from './errors.js';
40import { fail, plyError, argsError } from './errors.js';
41import type { Option, Result } from './result.js';
42import { ok, err, some, none, assertNever } from './result.js';
43import { withDefaults, fromNullish, mapGet } from './absent.js';
44import type { CheckedLimits, LimitsInput } from './limits.js';
45import { DEFAULT_LIMITS, LIMIT_WORK_RATIO, LIMIT_RESOLVE_FACTOR, checkLimit, limitExceeded, parseLimits } from './limits.js';
46import type { Oid } from './sha1.js';
47import { fromHex, toOid } from './sha1.js';
48import type { ScannedPack, ScannedType, ScanIndexOpts } from './packscanindex.js';
49import type { RangeSource, Halt } from './rangesource.js';
50import { UNCANCELLABLE, HALTED, haltOn, drop, cancelLate, aborted } from './rangesource.js';
51import { inflateInit, inflateFeed, inflatePull, inflateEnd, declaredBound, zlibWrap } from './inflatestream.js';
52import { applyDelta } from './packresolve.js';
53import { TYPE, TYPE_NAME, hashObject } from './objects.js';
55type ScanBudget = ScanIndexOpts['budget'];
56// packstream.ts WORK_BUDGET (the scan's default). Here its tag selects the bound from declared sizes (top of this file);
57// its ratio is the scan's and is only checked.
58const WORK_BUDGET: ScanBudget = Object.freeze({ tag: 'work', ratio: LIMIT_WORK_RATIO, capObjects: false });
59const OID_BYTES = 20;
60const { COMMIT: OBJ_COMMIT, TREE: OBJ_TREE, BLOB: OBJ_BLOB, TAG: OBJ_TAG, OFS_DELTA: OBJ_OFS_DELTA, REF_DELTA: OBJ_REF_DELTA } = TYPE;
62// In-range indexed reads of columns this module sized; noUncheckedIndexedAccess cannot see that.
63/* oxlint-disable typescript/no-unsafe-type-assertion, typescript/non-nullable-type-assertion-style -- in-range indexed reads, see above */
65export type ThinBase = { readonly type: number; readonly data: Uint8Array };
66// A thin pack's base, from the caller: the object stored under the oid asked for, or null when it has none.
67export type ThinBaseLookup = (oid: Oid) => Promise<ThinBase | null> | ThinBase | null;
68export type ResolveRangeOpts = {
69 readonly limits: LimitsInput;
70 readonly budget: ScanBudget;
71 // bytes of resolved bases kept for the deltas resting on them; a base not kept is rebuilt from its chain when needed
72 readonly cacheBytes: number;
73 readonly bases: ThinBaseLookup | null;
74 // aborts the resolve: every await on `source` and on `bases` is raced against it, so the call settles `aborted` at once
75 // even when the storage never answers (the range in flight is cancelled, a late answer dropped). Default: never aborts.
76 readonly signal: AbortSignal;
77};
78export const RESOLVE_CACHE_BYTES: number = 16 * 1024 * 1024;
79const defaultOpts = (): ResolveRangeOpts => ({ limits: DEFAULT_LIMITS, budget: WORK_BUDGET, cacheBytes: RESOLVE_CACHE_BYTES, bases: null, signal: UNCANCELLABLE });
81// oid 20 + type 1 + base 4 + depth 4 + size 8 + byOid 4
82export const RESOLVED_ENTRY_BYTES = 41;
83// `base`: the entry a delta was applied to; NO_BASE for a whole object; THIN_BASE for a base from `bases`.
84export const NO_BASE = -1;
85export const THIN_BASE = -2;
86export type ResolvedColumns = {
87 readonly oid: Uint8Array;
88 readonly type: Uint8Array;
89 readonly base: Int32Array;
90 readonly depth: Int32Array;
91 // the resolved object's size in bytes (a delta's result, not its instructions)
92 readonly size: Float64Array;
93};
94export type ResolveStats = {
95 // bytes inflated from the source, delta results produced, range reads made, bytes read back
96 readonly inflated: number;
97 readonly applied: number;
98 readonly reads: number;
99 readonly readBytes: number;
100 readonly cacheHits: number;
101 // bases made again (read back or rebuilt over their chain) because the cache did not hold them
102 readonly rebuilt: number;
103 readonly thinBases: number;
104 // the work charged against the budget (bytes inflated + bytes of delta results + one per child visited), and the budget
105 readonly work: number;
106 readonly workMax: number;
107};
108export type ResolvedIndex = {
109 readonly count: number;
110 readonly columns: ResolvedColumns;
111 readonly heldBytes: number;
112 readonly stats: ResolveStats;
113 readonly oid: (index: number) => Option<Oid>;
114 readonly type: (index: number) => Option<ScannedType>;
115 // the entry holding `oid` (the first, if the pack holds it twice), or none
116 readonly find: (oid: Oid) => Option<number>;
117};
119const isWhole = (t: number): t is ScannedType => t === OBJ_COMMIT || t === OBJ_TREE || t === OBJ_BLOB || t === OBJ_TAG;
120function oidCompare(a: Uint8Array, ai: number, b: Uint8Array, bi: number): number {
121 for (let k = 0; k < OID_BYTES; k++) {
122 const d = (a[ai + k] as number) - (b[bi + k] as number);
123 if (d !== 0) return d;
124 }
125 return 0;
128// Small LRU of resolved bases by entry index, held to `cap` bytes. A base larger than the cap is not kept.
129type Cache = { readonly map: Map<number, Uint8Array>; bytes: number; readonly cap: number };
130function cacheGet(c: Cache, i: number): Uint8Array | null {
131 const v = mapGet(c.map, i);
132 if (v.tag === 'none') return null;
133 c.map.delete(i);
134 c.map.set(i, v.value);
135 return v.value;
137function cachePut(c: Cache, i: number, v: Uint8Array): void {
138 if (v.length > c.cap || c.map.has(i)) return;
139 c.map.set(i, v);
140 c.bytes += v.length;
141 for (const [k, old] of c.map) {
142 if (c.bytes <= c.cap) break;
143 c.map.delete(k);
144 c.bytes -= old.length;
145 }
148type Stats = { inflated: number; applied: number; reads: number; readBytes: number; cacheHits: number; rebuilt: number; thinBases: number };
149type State = {
150 readonly pack: ScannedPack;
151 readonly source: RangeSource;
152 readonly bases: ThinBaseLookup | null;
153 readonly halt: Halt;
154 readonly n: number;
155 readonly maxObject: number;
156 readonly maxDelta: number;
157 readonly maxChain: number;
158 // the work bound: min(ceiling, max(flat, factor x floor)); floor grows by each delta result made for the first time
159 readonly flat: number;
160 readonly factor: number;
161 readonly ceiling: number;
162 floor: number;
163 work: number;
164 // the result the next entry popped rests on, held outside the cache (always the top of the stack, or null)
165 pin: { readonly index: number; readonly data: Uint8Array } | null;
166 readonly oid: Uint8Array;
167 readonly known: Uint8Array;
168 readonly type: Uint8Array;
169 readonly base: Int32Array;
170 readonly depth: Int32Array;
171 readonly size: Float64Array;
172 // OFS children of entry i: ofsKids[ofsStart[i] .. ofsStart[i + 1])
173 readonly ofsStart: Int32Array;
174 readonly ofsKids: Int32Array;
175 // REF deltas in order of their base id: refEntry[k]'s base id is refOid[k * 20 ..]
176 readonly refEntry: Int32Array;
177 readonly refOid: Uint8Array;
178 // refDone[lo] = 1 once the REF run starting at lo (one base id) has been applied
179 readonly refDone: Uint8Array;
180 // a delta resting on a thin base: thinOf[i] is that base's index in thinOids
181 readonly thinOf: Int32Array;
182 thinOids: Uint8Array;
183 thinCount: number;
184 readonly cache: Cache;
185 readonly stats: Stats;
186};
188const workMaxOf = (s: State): number => Math.min(s.ceiling, Math.max(s.flat, s.factor * s.floor));
189function charge(s: State, bytes: number): PlyError | null {
190 s.work += bytes;
191 const max = workMaxOf(s);
192 return s.work > max ? limitExceeded('resolve work bytes', s.work, max) : null;
194// A source (or a thin-base lookup) that throws, rejects or errors its stream failed the read: a coded error, never a
195// rejection of resolvePackRange.
196const readFailed = (what: string, e: unknown): PlyError => plyError('network', `pack: ${what} failed: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
198const stopped = (): PlyError => aborted('pack resolve');
199// The zlib member of entry i, inflated to its declared size, read back through the source.
200// Every await on the source goes through s.halt.race (rangesource.ts): an aborted resolve does not wait for it.
201async function readMember(s: State, i: number): Promise<Result<Uint8Array, PlyError>> {
202 const cols = s.pack.columns;
203 const at = (cols.offset[i] as number) + (cols.headerBytes[i] as number);
204 const length = cols.compressed[i] as number;
205 const size = cols.size[i] as number;
206 const charged = charge(s, size);
207 if (charged !== null) return err(charged);
208 const opened = inflateInit({ bound: declaredBound(size, isWhole(cols.storedType[i] as number) ? s.maxObject : s.maxDelta), wrap: zlibWrap(true) });
209 if (!opened.ok) return opened;
210 const st = opened.value;
211 let reader: ReadableStreamDefaultReader<Uint8Array>;
212 try {
213 const got = await s.halt.race(s.source.read(at, length, s.halt.signal), cancelLate);
214 if (got === HALTED) return err(stopped());
215 if (!got.ok) return got;
216 reader = got.value.getReader();
217 } catch (e) {
218 return err(readFailed(`range read of entry ${i}`, e));
219 }
220 s.stats.reads += 1;
221 s.halt.reading(reader);
222 const out = new Uint8Array(size);
223 let o = 0;
224 let fed = 0;
225 let ended = false;
226 try {
227 for (;;) {
228 const piece = ended ? inflateEnd(st) : inflatePull(st);
229 if (!piece.ok) return piece;
230 const p = piece.value;
231 if (p.kind === 'chunk') {
232 out.set(p.bytes, o);
233 o += p.bytes.length;
234 continue;
235 }
236 if (p.kind === 'end') {
237 if (p.consumed !== length || fed !== length) return fail('pack_corrupt', `pack: entry ${i} zlib member is ${p.consumed} of ${fed} bytes read, the index says ${length}`);
238 s.stats.inflated += o;
239 return ok(out);
240 }
241 let r: ReadableStreamReadResult<Uint8Array>;
242 try {
243 // oxlint-disable-next-line no-await-in-loop -- one member, read in order
244 const got = await s.halt.race(reader.read(), drop);
245 if (got === HALTED) return err(stopped());
246 r = got;
247 } catch (e) {
248 return err(readFailed(`range read of entry ${i}`, e));
249 }
250 if (r.done) {
251 ended = true;
252 continue;
253 }
254 fed += r.value.length;
255 s.stats.readBytes += r.value.length;
256 if (fed > length) return fail('pack_corrupt', `pack: range source gave more than ${length} bytes for entry ${i}`);
257 inflateFeed(st, r.value);
258 }
259 } finally {
260 // an aborted resolve has already asked the source to stop (the halt), and does not wait for it; otherwise it is
261 // asked here, and waited for only until an abort
262 if (!s.halt.halted()) {
263 s.halt.reading(null);
264 await s.halt.race(reader.cancel().catch(drop), drop);
265 }
266 }
269// type + bytes -> its oid, written at into[at ..]
270function hashTo(type: ScannedType, data: Uint8Array, into: Uint8Array, at: number): void {
271 into.set(fromHex(hashObject(TYPE_NAME[type], data)), at);
273const oidAt = (bytes: Uint8Array, i: number): Oid => toOid(bytes.subarray(i * OID_BYTES, (i + 1) * OID_BYTES));
274// A base made again must hash to the oid it had: anything else is the spool, not the pack.
275function checkAgain(s: State, i: number, data: Uint8Array): PlyError | null {
276 const scratch = new Uint8Array(OID_BYTES);
277 hashTo(s.type[i] as ScannedType, data, scratch, 0);
278 return oidCompare(scratch, 0, s.oid, i * OID_BYTES) === 0 ? null : plyError('pack_corrupt', `pack: entry ${i} read back as ${toOid(scratch)}, was ${oidAt(s.oid, i)}`);
281type Thin = { readonly tag: 'absent' } | { readonly tag: 'found'; readonly type: ScannedType; readonly data: Uint8Array };
282// The caller's object for `want`: absent, or checked (a stored type, and its bytes hash to `want`).
283async function thinData(s: State, want: Oid): Promise<Result<Thin, PlyError>> {
284 // the caller's lookup is its storage too
285 let found: ThinBase | null;
286 try {
287 const looked = s.bases === null ? null : await s.halt.race(Promise.resolve(s.bases(want)), drop);
288 if (looked === HALTED) return err(stopped());
289 found = looked;
290 } catch (e) {
291 return err(readFailed(`thin base lookup of ${want}`, e));
292 }
293 const asked = fromNullish(found);
294 if (asked.tag === 'none') return ok({ tag: 'absent' });
295 const got = asked.value;
296 if (!isWhole(got.type)) return fail('pack_corrupt', `pack: ref delta base ${want} has type ${got.type}`);
297 const hashed = hashObject(TYPE_NAME[got.type], got.data);
298 if (hashed !== want) return fail('pack_corrupt', `pack: ref delta base ${want} hashes to ${hashed}`);
299 return ok({ tag: 'found', type: got.type, data: got.data });
302// Entry i's delta, read back and applied to `base`, the result charged as work. `first`: the entry is being resolved
303// (not made again), so its result joins the floor the budget is measured against.
304async function applyEntry(s: State, i: number, base: Uint8Array, first: boolean): Promise<Result<Uint8Array, PlyError>> {
305 const delta = await readMember(s, i);
306 if (!delta.ok) return delta;
307 const applied = applyDelta(base, delta.value, { maxResult: s.maxDelta });
308 if (!applied.ok) return applied;
309 const out = applied.value.out;
310 if (first) s.floor += out.length;
311 const charged = charge(s, out.length);
312 if (charged !== null) return err(charged);
313 s.stats.applied += out.length;
314 return ok(out);
316// Entry i's resolved bytes: from the cache, read whole, or rebuilt over its base (which may itself be rebuilt). Each
317// frame holds nothing while its base is made, so the peak is one base and one result whatever the depth.
318async function materialize(s: State, i: number): Promise<Result<Uint8Array, PlyError>> {
319 const hit = cacheGet(s.cache, i);
320 if (hit !== null) {
321 s.stats.cacheHits += 1;
322 return ok(hit);
323 }
324 s.stats.rebuilt += 1;
325 let made: Result<Uint8Array, PlyError>;
326 const b = s.base[i] as number;
327 if (isWhole(s.pack.columns.storedType[i] as number)) made = await readMember(s, i);
328 else if (b === THIN_BASE) {
329 const want = oidAt(s.thinOids, s.thinOf[i] as number);
330 const thin = await thinData(s, want);
331 if (!thin.ok) return thin;
332 if (thin.value.tag === 'absent') return fail('pack_corrupt', `pack: ref delta base ${want} not found when asked again`);
333 made = await applyEntry(s, i, thin.value.data, false);
334 } else {
335 const base = await materialize(s, b);
336 if (!base.ok) return base;
337 // A base made again is kept (the LRU decides for how long): the siblings waiting behind this entry rest on the
338 // bases just below it, so each of them then costs one delta, not the chain from its root again.
339 cachePut(s.cache, b, base.value);
340 made = await applyEntry(s, i, base.value, false);
341 }
342 if (!made.ok) return made;
343 const bad = checkAgain(s, i, made.value);
344 return bad === null ? made : err(bad);
347// REF deltas whose base id is ids[at .. at + 20]: [lo, hi) in refEntry.
348function refRange(s: State, ids: Uint8Array, at: number): readonly [number, number] {
349 const m = s.refEntry.length;
350 let lo = 0;
351 let hi = m;
352 while (lo < hi) {
353 const mid = (lo + hi) >>> 1;
354 if (oidCompare(s.refOid, mid * OID_BYTES, ids, at) < 0) lo = mid + 1;
355 else hi = mid;
356 }
357 // the run's end by a second binary search, not a walk: m deltas on one base would walk the run m times (#299)
358 let end = lo;
359 hi = m;
360 while (end < hi) {
361 const mid = (end + hi) >>> 1;
362 if (oidCompare(s.refOid, mid * OID_BYTES, ids, at) <= 0) end = mid + 1;
363 else hi = mid;
364 }
365 return [lo, end];
367function hasKids(s: State, i: number): boolean {
368 if ((s.ofsStart[i + 1] as number) > (s.ofsStart[i] as number)) return true;
369 const [lo, hi] = refRange(s, s.oid, i * OID_BYTES);
370 return hi > lo && s.refDone[lo] === 0;
373// A base whose deltas are being applied: its entry (THIN_BASE for a thin base), type, depth, bytes, and its id at
374// ids[idAt ..].
375type Parent = { readonly index: number; readonly type: ScannedType; readonly depth: number; readonly data: Uint8Array; readonly ids: Uint8Array; readonly idAt: number };
376// Every not-yet-resolved delta resting on `p`, applied to its bytes: each gets its oid, type, depth and base; one that
377// something rests on is pushed and kept: the last such is pinned (it is popped next), the earlier ones go to the cache.
378async function resolveKids(s: State, p: Parent, stack: number[]): Promise<PlyError | null> {
379 const kids: number[] = [];
380 if (p.index >= 0) for (let k = s.ofsStart[p.index] as number; k < (s.ofsStart[p.index + 1] as number); k++) kids.push(s.ofsKids[k] as number);
381 const [lo, hi] = refRange(s, p.ids, p.idAt);
382 // A base id's REF run is applied once: every delta in it rests on whichever entry holding that id comes first, so a
383 // second holder (a duplicate, or a delta whose result is its base's own bytes) finds nothing to do (#299).
384 if (hi > lo && s.refDone[lo] === 0) {
385 s.refDone[lo] = 1;
386 for (let k = lo; k < hi; k++) kids.push(s.refEntry[k] as number);
387 }
388 const depth = p.depth + 1;
389 for (const c of kids) {
390 // every child looked at is charged: the work counts the visits as well as the bytes, so a shape that visits a delta
391 // more than a constant number of times is refused by the budget, not resolved slowly
392 const visited = charge(s, 1);
393 if (visited !== null) return visited;
394 if (s.known[c] === 1) continue;
395 if (depth > s.maxChain) return limitExceeded('delta chain depth', depth, s.maxChain);
396 // oxlint-disable-next-line no-await-in-loop -- each child is applied in turn against the one base held
397 const out = await applyEntry(s, c, p.data, true);
398 if (!out.ok) return out.error;
399 hashTo(p.type, out.value, s.oid, c * OID_BYTES);
400 s.known[c] = 1;
401 s.type[c] = p.type;
402 s.depth[c] = depth;
403 s.size[c] = out.value.length;
404 s.base[c] = p.index;
405 if (hasKids(s, c)) {
406 if (s.pin !== null) cachePut(s.cache, s.pin.index, s.pin.data);
407 s.pin = { index: c, data: out.value };
408 stack.push(c);
409 }
410 }
411 return null;
413// Depth first: each entry popped is made (the pinned result, else a cache hit, else rebuilt) and the deltas resting on
414// it applied.
415async function drain(s: State, stack: number[]): Promise<PlyError | null> {
416 while (stack.length > 0) {
417 const u = stack.pop() as number;
418 const pinned = s.pin;
419 s.pin = null;
420 // oxlint-disable-next-line no-await-in-loop -- one chain at a time, in order
421 const data = pinned !== null && pinned.index === u ? ok(pinned.data) : await materialize(s, u);
422 if (!data.ok) return data.error;
423 const p: Parent = { index: u, type: s.type[u] as ScannedType, depth: s.depth[u] as number, data: data.value, ids: s.oid, idAt: u * OID_BYTES };
424 // oxlint-disable-next-line no-await-in-loop -- as above
425 const bad = await resolveKids(s, p, stack);
426 if (bad !== null) return bad;
427 }
428 return null;
431// OFS children by base index (counting sort), REF deltas sorted by base id, whole oids the scan computed.
432function setup(pack: ScannedPack, source: RangeSource, opts: ResolveRangeOpts, caps: { readonly maxObject: number; readonly maxDelta: number; readonly maxChain: number; readonly flat: number; readonly factor: number; readonly ceiling: number }, halt: Halt): State {
433 const cols = pack.columns;
434 const n = cols.offset.length;
435 const ofsStart = new Int32Array(n + 1);
436 let refs = 0;
437 for (let i = 0; i < n; i++) {
438 const t = cols.storedType[i] as number;
439 if (t === OBJ_OFS_DELTA) {
440 const at = (cols.baseIndex[i] as number) + 1;
441 ofsStart[at] = (ofsStart[at] as number) + 1;
442 } else if (t === OBJ_REF_DELTA) refs += 1;
443 }
444 for (let i = 0; i < n; i++) ofsStart[i + 1] = (ofsStart[i + 1] as number) + (ofsStart[i] as number);
445 const ofsKids = new Int32Array(ofsStart[n] as number);
446 const fill = ofsStart.slice(0, n);
447 const refAt = new Int32Array(refs);
448 const refRaw = new Uint8Array(refs * OID_BYTES);
449 const oid = new Uint8Array(n * OID_BYTES);
450 const known = new Uint8Array(n);
451 let r = 0;
452 for (let i = 0; i < n; i++) {
453 const t = cols.storedType[i] as number;
454 if (t === OBJ_OFS_DELTA) {
455 const b = cols.baseIndex[i] as number;
456 ofsKids[fill[b] as number] = i;
457 fill[b] = (fill[b] as number) + 1;
458 continue;
459 }
460 const e = pack.entry(i);
461 if (e.tag === 'none') continue;
462 if (t === OBJ_REF_DELTA && e.value.base.tag === 'ref') {
463 refRaw.set(fromHex(e.value.base.oid), r * OID_BYTES);
464 refAt[r] = i;
465 r += 1;
466 } else if (e.value.oid !== null) {
467 oid.set(fromHex(e.value.oid), i * OID_BYTES);
468 known[i] = 1;
469 }
470 }
471 // REF deltas by base id, pack order among equal ids
472 const order = Int32Array.from({ length: refs }, (_, k) => k).toSorted((x, y) => oidCompare(refRaw, x * OID_BYTES, refRaw, y * OID_BYTES) || x - y);
473 const refOid = new Uint8Array(refs * OID_BYTES);
474 const refEntry = new Int32Array(refs);
475 for (let k = 0; k < refs; k++) {
476 const from = order[k] as number;
477 refOid.set(refRaw.subarray(from * OID_BYTES, (from + 1) * OID_BYTES), k * OID_BYTES);
478 refEntry[k] = refAt[from] as number;
479 }
480 return {
481 pack, source, bases: opts.bases, halt, n, ...caps, floor: cols.size.reduce((a, b) => a + b, 0), work: 0, pin: null, oid, known,
482 type: new Uint8Array(n), base: new Int32Array(n).fill(NO_BASE), depth: new Int32Array(n), size: Float64Array.from(pack.columns.size), ofsStart, ofsKids,
483 refEntry, refOid, refDone: new Uint8Array(refs), thinOf: new Int32Array(n).fill(-1), thinOids: new Uint8Array(0), thinCount: 0,
484 cache: { map: new Map(), bytes: 0, cap: opts.cacheBytes },
485 stats: { inflated: 0, applied: 0, reads: 0, readBytes: 0, cacheHits: 0, rebuilt: 0, thinBases: 0 },
486 };
489// One whole object as a root: its oid (hashed now if the scan had oids off), then everything resting on it.
490async function fromRoot(s: State, i: number, t: ScannedType): Promise<PlyError | null> {
491 s.type[i] = t;
492 const hashedBefore = s.known[i] === 1;
493 if (hashedBefore && !hasKids(s, i)) return null;
494 const data = hashedBefore ? await materialize(s, i) : await readMember(s, i);
495 if (!data.ok) return data.error;
496 if (!hashedBefore) {
497 hashTo(t, data.value, s.oid, i * OID_BYTES);
498 s.known[i] = 1;
499 if (!hasKids(s, i)) return null;
500 }
501 cachePut(s.cache, i, data.value);
502 const stack: number[] = [];
503 const kids = await resolveKids(s, { index: i, type: t, depth: 0, data: data.value, ids: s.oid, idAt: i * OID_BYTES }, stack);
504 return kids === null ? drain(s, stack) : kids;
506async function fromRoots(s: State): Promise<PlyError | null> {
507 const stored = s.pack.columns.storedType;
508 for (let i = 0; i < s.n; i++) {
509 const t = stored[i] as number;
510 if (!isWhole(t)) continue;
511 // oxlint-disable-next-line no-await-in-loop -- one root at a time
512 const bad = await fromRoot(s, i, t);
513 if (bad !== null) return bad;
514 }
515 return null;
518type ThinStep = { readonly tag: 'absent' } | { readonly tag: 'resolved' } | { readonly tag: 'failed'; readonly error: PlyError };
519// The id at refOid[k] asked of `bases`: absent, or a root whose deltas (and theirs) are resolved now.
520async function thinRoot(s: State, k: number): Promise<ThinStep> {
521 const id = s.refOid.subarray(k * OID_BYTES, (k + 1) * OID_BYTES);
522 const thin = await thinData(s, toOid(id));
523 if (!thin.ok) return { tag: 'failed', error: thin.error };
524 if (thin.value.tag === 'absent') return { tag: 'absent' };
525 const t = s.thinCount;
526 const grown = new Uint8Array((t + 1) * OID_BYTES);
527 grown.set(s.thinOids);
528 grown.set(id, t * OID_BYTES);
529 s.thinOids = grown;
530 s.thinCount += 1;
531 s.stats.thinBases += 1;
532 const [lo, hi] = refRange(s, s.refOid, k * OID_BYTES);
533 for (let j = lo; j < hi; j++) s.thinOf[s.refEntry[j] as number] = t;
534 const stack: number[] = [];
535 const kids = await resolveKids(s, { index: THIN_BASE, type: thin.value.type, depth: 0, data: thin.value.data, ids: s.refOid, idAt: k * OID_BYTES }, stack);
536 const bad = kids === null ? await drain(s, stack) : kids;
537 return bad === null ? { tag: 'resolved' } : { tag: 'failed', error: bad };
539// After the pack's own roots: a REF delta still unresolved names a base outside the pack, or an in-pack delta waiting
540// on one. Each such id is asked of `bases` once; one it has becomes a root. Rounds repeat while one makes progress (an
541// id asked too early may name an in-pack delta resolved later). Then the first entry left is the refusal.
542async function fromThin(s: State): Promise<PlyError | null> {
543 const asked = new Set<string>();
544 for (let progress = s.bases !== null; progress;) {
545 progress = false;
546 for (let k = 0; k < s.refEntry.length; k++) {
547 if (s.known[s.refEntry[k] as number] === 1) continue;
548 const hex = oidAt(s.refOid, k);
549 if (asked.has(hex)) continue;
550 asked.add(hex);
551 // oxlint-disable-next-line no-await-in-loop -- one thin base at a time
552 const step = await thinRoot(s, k);
553 switch (step.tag) {
554 case 'failed': return step.error;
555 case 'resolved': progress = true; continue;
556 case 'absent': continue;
557 }
558 // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check
559 return assertNever(step);
560 }
561 }
562 for (let i = 0; i < s.n; i++) {
563 if (s.known[i] === 1) continue;
564 const e = s.pack.entry(i);
565 const named = e.tag === 'some' && e.value.base.tag === 'ref' ? e.value.base.oid : `at entry ${s.pack.columns.baseIndex[i] as number}`;
566 const hint = s.bases === null ? ' (a thin pack needs bases)' : '';
567 return plyError('pack_corrupt', `pack: delta at entry ${i} unresolved: ref delta base ${named} not found${hint}`);
568 }
569 return null;
572// An index over resolved `columns` (each sized to the entries) with the stats of the run that produced them.
573// resolvePackRange builds one after resolving; decodeSavedIndex (packsaved.ts, #260) builds one from saved columns.
574export function resolvedIndexOf(columns: ResolvedColumns, stats: ResolveStats): ResolvedIndex {
575 const n = columns.type.length;
576 const ids = columns.oid;
577 const byOid = Int32Array.from({ length: n }, (_, i) => i).toSorted((x, y) => oidCompare(ids, x * OID_BYTES, ids, y * OID_BYTES) || x - y);
578 const inRange = (i: number): boolean => Number.isInteger(i) && i >= 0 && i < n;
579 const find = (want: Oid): Option<number> => {
580 if (!/^[0-9a-f]{40}$/u.test(want)) return none;
581 const id = fromHex(want);
582 let lo = 0;
583 let hi = n;
584 while (lo < hi) {
585 const mid = (lo + hi) >>> 1;
586 if (oidCompare(ids, (byOid[mid] as number) * OID_BYTES, id, 0) < 0) lo = mid + 1;
587 else hi = mid;
588 }
589 return lo < n && oidCompare(ids, (byOid[lo] as number) * OID_BYTES, id, 0) === 0 ? some(byOid[lo] as number) : none;
590 };
591 return {
592 count: n,
593 columns,
594 heldBytes: n * RESOLVED_ENTRY_BYTES,
595 stats,
596 oid: i => (inRange(i) ? some(oidAt(ids, i)) : none),
597 type: i => (inRange(i) ? some(columns.type[i] as ScannedType) : none),
598 find,
599 };
601function finish(s: State): ResolvedIndex {
602 return resolvedIndexOf({ oid: s.oid, type: s.type, base: s.base, depth: s.depth, size: s.size }, { ...s.stats, work: s.work, workMax: workMaxOf(s) });
605// The options a resolve runs under: each key read from the caller's record as input[key] (withDefaults), so an inherited
606// property or an accessor counts as an own one does, and the limits parsed. loadTransferPack (transferload.ts) holds a
607// saved index to this record's limits.maxDeltaChain and hands the record on to resolvePackRange, which reads it again.
608type CheckedOpts = Omit<ResolveRangeOpts, 'limits'> & { readonly limits: CheckedLimits };
609export function resolveOptsOf(input: Partial<ResolveRangeOpts>): Result<CheckedOpts, PlyError> {
610 const opts = withDefaults(input, defaultOpts());
611 const limits = parseLimits(opts.limits);
612 return limits.ok ? ok({ ...opts, limits: limits.value }) : limits;
615// Resolve every entry of `pack` (scanPackIndex's index of the bytes `source` serves). See the top of this file.
616export async function resolvePackRange(pack: ScannedPack, source: RangeSource, input: Partial<ResolveRangeOpts> = {}): Promise<Result<ResolvedIndex, PlyError>> {
617 const shape = argsError('resolvePackRange', [['pack', pack, 'object'], ['source', source, 'object'], ['options', input, 'object']]);
618 if (shape !== null) return shape;
619 const parsed = resolveOptsOf(input);
620 if (!parsed.ok) return parsed;
621 const opts = parsed.value;
622 // aborted before the call: nothing is asked of the storage. A pack of whole objects needs no read, so no later await
623 // would see the abort.
624 if (opts.signal.aborted) return err(stopped());
625 if (source.size !== pack.bytes) return fail('bad_request', `range source is ${source.size} bytes, the index read ${pack.bytes}`);
626 if (!Number.isFinite(opts.cacheBytes) || opts.cacheBytes < 0) return fail('bad_request', `cacheBytes must be a non-negative number, got ${opts.cacheBytes}`);
627 const budget = opts.budget;
628 const work = budget.tag === 'work';
629 if (work) {
630 const ratio = checkLimit(budget.ratio, 'workRatio');
631 if (!ratio.ok) return ratio;
632 }
633 const l = opts.limits;
634 const flat = l.maxInflatedTotal;
635 // the ceiling is the parsed maxResolveWork (parseLimits: never below maxInflatedTotal)
636 const bound = work ? { flat, factor: LIMIT_RESOLVE_FACTOR, ceiling: l.maxResolveWork } : { flat, factor: 0, ceiling: flat };
637 const halt = haltOn(opts.signal);
638 try {
639 const s = setup(pack, source, opts, { maxObject: l.maxObjectSize, maxDelta: l.maxDeltaResult, maxChain: l.maxDeltaChain, ...bound }, halt);
640 const roots = await fromRoots(s);
641 if (roots !== null) return err(roots);
642 const thin = await fromThin(s);
643 return thin === null ? ok(finish(s)) : err(thin);
644 } finally {
645 halt.release();
646 }
Diagnostics6 git calls · 76 ms
CallStatusSizeTime076 ms
discover200191 B1 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit1 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=1
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=1
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02002.4 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=69
git.response.bytes=2507
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2507
git.pkt.payload.max=2480
git.sideband.band1.bytes=2480
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=2480
git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381
Open pack map
fetch20010 KiB69 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=10544
git.response.cache=MISS
git.response.duration_ms=69
git.fetch.wants=956d712b1d639d5a31c86a07828e3685217c90db
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=5
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=10544
git.pkt.payload.max=8192
git.sideband.band1.bytes=10512
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0f84b2c50332c
Request
d4ffb839…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format