GitHubivarvong
ply
main
1// resolvePackRange (#248): every entry's oid, type and chain depth, applied over a RangeSource. Held to `git verify-pack
2// -v` on packs git wrote (this repository's objects, REF-only as Artifacts sends them and with OFS deltas), to
3// resolvePack (the buffered resolver) on a thin pack with its bases, and to itself across cache sizes and scans with
4// oids on and off. Then the refusals: a thin pack without bases, a REF cycle, a chain past maxDeltaChain, work past the
5// budget, a delta against the wrong base size, a spool that changed between reads, a source of the wrong size.
6import { test } from 'vitest';
7import assert from 'node:assert/strict';
8import { execFileSync } from 'node:child_process';
9import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
10import { join } from 'node:path';
11import { tmpdir } from 'node:os';
12import { inflateSync } from 'node:zlib';
13import { scanPackIndex } from '../src/packscanindex.js';
14import { resolvePackRange, RESOLVED_ENTRY_BYTES, NO_BASE, THIN_BASE } from '../src/packrangeresolve.js';
15import { rangeFromBytes } from '../src/rangesource.js';
16import { parsePack, resolvePack, encodeObjHeader, zlibDeflate } from '../src/pack.js';
17import { hashObject, TYPE } from '../src/objects.js';
18import { Sha1, fromHex } from '../src/sha1.js';
19import { concat, te } from '../src/bytes.js';
20import { withLimits, LIMIT_RESOLVE_WORK, LIMIT_INFLATED_TOTAL } from '../src/limits.js';
21import { ok } from '../src/result.js';
22import { RETAIN_BUDGET } from '../src/packstream.js';
23import { PACK_SIGNATURE, PACK_VERSION, PACK_HEADER_BYTES, UINT32_BYTES, OBJ_CONTINUE, OBJ_SIZE_NEXT_MASK, OBJ_SIZE_NEXT_BASE } from '../src/format.js';
24import { buildPacks } from './space/pack-fixtures.js';
25import { settledWithin, stallAt, abortedBefore, codeOf } from './abortcells.js';
27const must = r => { if (r.ok) return r.value; throw Object.assign(new Error(r.error.message), { code: r.error.code }); };
28const streamOf = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
29const scan = (pack, oids = true) => scanPackIndex(streamOf(pack), { oids });
30async function resolved(pack, opts = {}, oids = true) {
31 return resolvePackRange(must(await scan(pack, oids)), rangeFromBytes(pack), opts);
32}
34const gitIn = (cwd, args, input) => execFileSync('git', args, { cwd, input, maxBuffer: 1 << 30 });
35// git verify-pack -v, by offset: oid, type, the stored size (a delta's own size for a delta) and the chain depth (0 for a
36// whole object).
37function verifyPack(pack) {
38 const dir = mkdtempSync(join(tmpdir(), 'ply-range-resolve-'));
39 try {
40 const file = join(dir, 'p.pack');
41 writeFileSync(file, pack);
42 gitIn(dir, ['index-pack', file]);
43 const out = gitIn(dir, ['verify-pack', '-v', join(dir, 'p.idx')]).toString();
44 const byOffset = new Map();
45 for (const line of out.split('\n')) {
46 const m = /^([0-9a-f]{40}) (commit|tree|blob|tag)\s+(\d+) \d+ (\d+)(?: (\d+) ([0-9a-f]{40}))?/.exec(line);
47 if (m) byOffset.set(Number(m[4]), { oid: m[1], type: TYPE[m[2].toUpperCase()], stored: Number(m[3]), depth: m[5] === undefined ? 0 : Number(m[5]), base: m[6] ?? null });
48 }
49 return byOffset;
50 } finally { rmSync(dir, { recursive: true, force: true }); }
51}
52// Every entry against git: oid, type, depth, and the base it was applied to.
53function assertMatchesGit(pack, idx, res) {
54 const git = verifyPack(pack);
55 assert.equal(git.size, res.count);
56 for (let i = 0; i < res.count; i++) {
57 const g = git.get(idx.columns.offset[i]);
58 assert.ok(g, `git has no object at ${idx.columns.offset[i]}`);
59 assert.equal(res.oid(i).value, g.oid, `oid of entry ${i}`);
60 assert.equal(res.type(i).value, g.type, `type of entry ${i}`);
61 assert.equal(res.columns.depth[i], g.depth, `depth of entry ${i}`);
62 const b = res.columns.base[i];
63 if (g.base === null) assert.equal(b, NO_BASE);
64 else assert.equal(res.oid(b).value, g.base, `base of entry ${i}`);
65 assert.equal(res.find(g.oid).value, i);
66 }
67}
68// This repository's history up to a fixed commit, not HEAD: the work a pack costs to resolve grows with the history
69// packed, so a pack of HEAD would turn these tests red at some length of history. The bound (src/packrangeresolve.ts
70// workMaxOf) is min(ceiling, max(maxInflatedTotal, LIMIT_RESOLVE_FACTOR x floor)), the floor being every member's
71// stored size plus every delta's result. On DEFAULT_LIMITS: ceiling = LIMIT_RESOLVE_WORK = LIMIT_INFLATED_TOTAL x
72// LIMIT_RESOLVE_TOTALS = 256 MiB x 8 = 2 GiB (2147.5 MB); maxInflatedTotal = LIMIT_OBJECT x LIMIT_INFLATED_OBJECTS =
73// 64 MiB x 4 = 256 MiB (268.4 MB); the factor is 4. For this pack the floor is about 70.6 MB, so 4 x floor is about
74// 282.4 MB, over 268.4 MB and under 2147.5 MB: the middle term sets the bound (the test below derives it from git).
75// Measured with git 2.50 at this commit (2026-11-11): the default cache does 69.2 MB of work and the 64 KiB cache
76// below 96.8 to 98.4 MB (pack-objects output varies a little run to run), about 35% of the bound. The work and the
77// floor both grow with the history, so the pin is what keeps this test's cost fixed.
78// Every full clone has the commit. A clone without it (a shallow one) packs its tip commit alone, which is all a
79// depth-1 clone holds and does not grow with the history either.
80const PACKED_COMMIT = 'ec337c5258839576e354ca9ea12d4f2404a3a75f';
81const repoPack = args => {
82 const held = gitIn(process.cwd(), ['cat-file', '--batch-check'], `${PACKED_COMMIT}\n`).toString().startsWith(`${PACKED_COMMIT} commit `);
83 const objects = gitIn(process.cwd(), ['rev-list', '--objects', ...(held ? [PACKED_COMMIT] : ['--max-count=1', 'HEAD'])]);
84 return new Uint8Array(gitIn(process.cwd(), ['pack-objects', '--stdout', '-q', '--depth=50', ...args], objects));
85};
87test("this repository's pack, REF_DELTA only (what Artifacts sends): every oid, type, depth and base matches git verify-pack", async () => {
88 const pack = repoPack([]);
89 const idx = must(await scan(pack));
90 assert.equal([...idx.columns.storedType].includes(TYPE.OFS_DELTA), false);
91 const unknownToScan = [...idx.columns.type].filter(t => t === 0).length;
92 const t0 = performance.now();
93 const res = must(await resolvePackRange(idx, rangeFromBytes(pack)));
94 const ms = performance.now() - t0;
95 assertMatchesGit(pack, idx, res);
96 const deepest = Math.max(...res.columns.depth);
97 console.log(JSON.stringify({ entries: res.count, packBytes: pack.length, unknownToScan, deepest, ms: Math.round(ms), mbPerS: +(pack.length / 1e6 / (ms / 1000)).toFixed(1), resolvedMBPerS: +(res.stats.applied / 1e6 / (ms / 1000)).toFixed(1), stats: res.stats }));
98 assert.ok(res.count > 100, `only ${res.count} objects`);
99 // CI's shallow checkout may hold no chain deeper than one; a full clone holds thousands
100 if (unknownToScan > 0) assert.ok(deepest > 1);
101 assert.equal(res.heldBytes, res.count * RESOLVED_ENTRY_BYTES);
102});
104test("this repository's pack with OFS deltas matches git verify-pack; scanned with oids off, the roots are hashed here", async () => {
105 const pack = repoPack(['--delta-base-offset']);
106 const idx = must(await scan(pack, false));
107 const res = must(await resolvePackRange(idx, rangeFromBytes(pack)));
108 assertMatchesGit(pack, idx, res);
109});
111// The work bound for this pack, from git: the floor is every member's stored size (verify-pack's size column, a delta's
112// own size for a delta) plus every delta's result (cat-file's size of each deltified object), each counted once.
113test("this repository's pack: the work bound is min(ceiling, max(maxInflatedTotal, 4 x floor)), the floor taken from git", async () => {
114 const pack = repoPack([]);
115 const git = [...verifyPack(pack).values()];
116 const deltas = git.filter(g => g.base !== null).map(g => g.oid);
117 const resultSizes = gitIn(process.cwd(), ['cat-file', '--batch-check=%(objectsize)'], deltas.join('\n') + '\n').toString().trim().split('\n').map(Number);
118 const floor = git.reduce((a, g) => a + g.stored, 0) + resultSizes.reduce((a, b) => a + b, 0);
119 const bound = Math.min(LIMIT_RESOLVE_WORK, Math.max(LIMIT_INFLATED_TOTAL, 4 * floor));
120 const idx = must(await scan(pack));
121 for (const cacheBytes of [undefined, 64 << 10]) {
122 const res = must(await resolvePackRange(idx, rangeFromBytes(pack), cacheBytes === undefined ? {} : { cacheBytes }));
123 assert.equal(res.stats.workMax, bound, `cache ${cacheBytes}`);
124 console.log(JSON.stringify({ cacheBytes: cacheBytes ?? 'default', floor, workMax: res.stats.workMax, work: res.stats.work }));
125 }
126});
128test('the same result at every cache size, down to none (every base rebuilt over its chain)', async () => {
129 const { packs } = buildPacks();
130 // With no cache a chain is rebuilt from its root for every entry popped: quadratic in depth, so this repository's
131 // pack (depth 50) runs at a small cache, not none. That cost is what the work budget bounds.
132 for (const [bytes, sizes] of [[packs.chain.bytes, [0, 1024, 1 << 20]], [packs.ref.bytes, [0, 1024]], [repoPack([]), [64 << 10, 1 << 20]]]) {
133 const idx = must(await scan(bytes));
134 const full = must(await resolvePackRange(idx, rangeFromBytes(bytes)));
135 for (const cacheBytes of sizes) {
136 const r = must(await resolvePackRange(idx, rangeFromBytes(bytes), { cacheBytes }));
137 assert.deepEqual(r.columns.oid, full.columns.oid);
138 assert.deepEqual(r.columns.type, full.columns.type);
139 assert.deepEqual(r.columns.depth, full.columns.depth);
140 assert.deepEqual(r.columns.base, full.columns.base);
141 assert.ok(r.stats.rebuilt >= full.stats.rebuilt, `cache ${cacheBytes}: rebuilt ${r.stats.rebuilt} < ${full.stats.rebuilt}`);
142 }
143 }
144});
146// Hand-built packs: { type, data } whole, { data, baseOid } REF_DELTA, { data, baseIndex } OFS_DELTA.
147function encodeOfs(off) {
148 const out = [off & OBJ_SIZE_NEXT_MASK];
149 while ((off = Math.floor(off / OBJ_SIZE_NEXT_BASE)) > 0) out.unshift(OBJ_CONTINUE | (--off & OBJ_SIZE_NEXT_MASK));
150 return Uint8Array.of(...out);
151}
152async function packOf(entries) {
153 const parts = [Uint8Array.of(...PACK_SIGNATURE, 0, 0, 0, PACK_VERSION), new Uint8Array(UINT32_BYTES)];
154 new DataView(parts[1].buffer).setUint32(0, entries.length);
155 const offsets = [];
156 let at = PACK_HEADER_BYTES;
157 for (const e of entries) {
158 offsets.push(at);
159 const ofs = e.baseIndex !== undefined ? at - offsets[e.baseIndex] : undefined;
160 const type = e.baseOid ? TYPE.REF_DELTA : ofs !== undefined ? TYPE.OFS_DELTA : e.type;
161 const entry = concat([encodeObjHeader(type, e.data.length), ...(e.baseOid ? [fromHex(e.baseOid)] : ofs !== undefined ? [encodeOfs(ofs)] : []), await zlibDeflate(e.data)]);
162 parts.push(entry);
163 at += entry.length;
164 }
165 const body = concat(parts);
166 return concat([body, new Sha1().update(body).digest()]);
167}
168const varint = n => { const out = []; do { let b = n & 0x7f; n >>>= 7; if (n) b |= 0x80; out.push(b); } while (n); return out; };
169// base -> target: copy the base's first `keep` bytes, insert the rest
170function editDelta(base, target, keep = 0) {
171 const ops = [];
172 if (keep > 0) ops.push(0x80 | 0x10, keep & 0xff);
173 for (let i = keep; i < target.length; i += 127) { const piece = target.subarray(i, i + 127); ops.push(piece.length, ...piece); }
174 return Uint8Array.of(...varint(base.length), ...varint(target.length), ...ops);
175}
176// a REF chain of `depth` blobs, written deepest first so every delta's base comes later in the pack
177function refChain(depth, seed = 'v') {
178 const versions = Array.from({ length: depth + 1 }, (_, i) => te.encode(`${seed} ${'x'.repeat(40)} version ${i}\n`));
179 const oids = versions.map(v => hashObject('blob', v));
180 const entries = [];
181 for (let i = depth; i >= 1; i--) entries.push({ data: editDelta(versions[i - 1], versions[i], 40), baseOid: oids[i - 1] });
182 entries.push({ type: TYPE.BLOB, data: versions[0] });
183 return { entries, versions, oids };
184}
186test('a REF chain 100 deep, every base later in the pack: resolved at maxDeltaChain 128, refused past maxDeltaChain', async () => {
187 const { entries, oids } = refChain(100);
188 const pack = await packOf(entries);
189 const idx = must(await scan(pack));
190 // the scan knows only the delta resting directly on the whole object
191 assert.equal([...idx.columns.type].filter(t => t === 0).length, 99);
192 const limits = withLimits({ maxDeltaChain: 128 });
193 for (const cacheBytes of [0, 16 << 20]) {
194 const res = must(await resolvePackRange(idx, rangeFromBytes(pack), { limits, cacheBytes }));
195 for (let i = 0; i < 100; i++) {
196 assert.equal(res.oid(i).value, oids[100 - i]);
197 assert.equal(res.columns.depth[i], 100 - i);
198 assert.equal(res.type(i).value, TYPE.BLOB);
199 }
200 assert.equal(res.oid(100).value, oids[0]);
201 assert.equal(res.columns.depth[100], 0);
202 }
203 const deep = await resolvePackRange(idx, rangeFromBytes(pack), { limits: withLimits({ maxDeltaChain: 64 }) });
204 assert.equal(deep.ok, false);
205 assert.equal(deep.error.code, 'limit_exceeded');
206 assert.equal(deep.error.what, 'delta chain depth');
207});
209test('a thin pack: refused without bases; with them, matches resolvePack (oid and chain at every offset)', async () => {
210 const { packs, bases } = buildPacks();
211 const pack = packs.thin.bytes;
212 const without = await resolved(pack);
213 assert.equal(without.ok, false);
214 assert.equal(without.error.code, 'pack_corrupt');
215 assert.match(without.error.message, /not found \(a thin pack needs bases\)/);
216 const lookup = async oid => { await Promise.resolve(); const b = bases.get(oid); return b ? { type: TYPE[b.type.toUpperCase()], data: b.data } : null; };
217 const res = must(await resolved(pack, { bases: lookup }));
218 const oracle = must(resolvePack(must(parsePack(pack)), oid => { const b = bases.get(oid); return b ? { type: TYPE[b.type.toUpperCase()], data: b.data } : null; }));
219 const idx = must(await scan(pack));
220 const byOffset = new Map([...oracle.objects.values()].map(o => [o.offset, o]));
221 for (let i = 0; i < res.count; i++) {
222 const o = byOffset.get(idx.columns.offset[i]);
223 assert.equal(res.oid(i).value, o.oid);
224 assert.equal(res.columns.depth[i], o.chain);
225 }
226 assert.ok([...res.columns.base].includes(THIN_BASE));
227 assert.ok(res.stats.thinBases > 0);
228 // a base whose bytes do not hash to the oid that named it is refused
229 const lying = await resolved(pack, { bases: oid => (bases.has(oid) ? { type: TYPE.BLOB, data: te.encode('not it') } : null) });
230 assert.equal(lying.error.code, 'pack_corrupt');
231 assert.match(lying.error.message, /hashes to/);
232});
234test('a thin base asked too early: the REF id named first is an in-pack delta that waits on the thin base', async () => {
235 // b rests (REF) on a; a rests (REF) on z, which is outside the pack. Sorted by base id, b may be asked first.
236 const z = te.encode(`zeta ${'z'.repeat(40)}\n`);
237 const a = te.encode(`zeta ${'z'.repeat(40)}\nalpha\n`);
238 const b = te.encode(`zeta ${'z'.repeat(40)}\nalpha\nbeta\n`);
239 const [zo, ao] = [hashObject('blob', z), hashObject('blob', a)];
240 const pack = await packOf([{ data: editDelta(a, b, 10), baseOid: ao }, { data: editDelta(z, a, 10), baseOid: zo }]);
241 const asked = [];
242 const res = must(await resolved(pack, { bases: oid => { asked.push(oid); return oid === zo ? { type: TYPE.BLOB, data: z } : null; } }));
243 assert.equal(res.oid(0).value, hashObject('blob', b));
244 assert.equal(res.oid(1).value, ao);
245 assert.equal(res.columns.base[1], THIN_BASE);
246 assert.equal(res.columns.base[0], 1);
247 assert.ok(asked.includes(zo));
248});
250test('refusals: a REF cycle, work past the budget, a delta against the wrong base size, a source of the wrong size, a bad cacheBytes', async () => {
251 // two deltas naming each other: neither oid can be known, so neither resolves
252 const x = te.encode('x'.repeat(50));
253 const cyc = await packOf([{ data: editDelta(x, x), baseOid: 'aa'.repeat(20) }, { data: editDelta(x, x), baseOid: 'bb'.repeat(20) }]);
254 const r1 = await resolved(cyc);
255 assert.equal(r1.error.code, 'pack_corrupt');
256 assert.match(r1.error.message, /unresolved: ref delta base [0-9a-f]{40} not found/);
257 // work: maxInflatedTotal is the budget when the pack is small; the chain needs more
258 const { entries } = refChain(30);
259 const chain = await packOf(entries);
260 const r2 = await resolved(chain, { limits: withLimits({ maxInflatedTotal: 2000 }), budget: RETAIN_BUDGET, cacheBytes: 0 });
261 assert.equal(r2.error.code, 'limit_exceeded');
262 assert.equal(r2.error.what, 'resolve work bytes');
263 // a delta whose declared base size is not its base's
264 const base = te.encode('base bytes here');
265 const wrong = await packOf([{ type: TYPE.BLOB, data: base }, { data: editDelta(te.encode('other length!'), base), baseIndex: 0 }]);
266 const r3 = await resolved(wrong);
267 assert.equal(r3.error.code, 'pack_corrupt');
268 assert.match(r3.error.message, /delta: base size/);
269 // the source is not the bytes the index read
270 const idx = must(await scan(chain));
271 const r4 = await resolvePackRange(idx, rangeFromBytes(chain.subarray(1)));
272 assert.equal(r4.error.code, 'bad_request');
273 const r5 = await resolvePackRange(idx, rangeFromBytes(chain), { cacheBytes: -1 });
274 assert.equal(r5.error.code, 'bad_request');
275});
277test('a spool that changed between reads: a base read back again with other bytes is pack_corrupt, not a wrong oid', async () => {
278 // a whole object with two OFS deltas resting on a third: with no cache, the whole object is read more than once
279 const v0 = te.encode(`version zero ${'q'.repeat(60)}\n`);
280 const v1 = te.encode(`version zero ${'q'.repeat(60)}\none\n`);
281 const v2 = te.encode(`version zero ${'q'.repeat(60)}\none\ntwo\n`);
282 const pack = await packOf([{ type: TYPE.BLOB, data: v0 }, { data: editDelta(v0, v1, 20), baseIndex: 0 }, { data: editDelta(v1, v2, 20), baseIndex: 1 }]);
283 const idx = must(await scan(pack));
284 const good = rangeFromBytes(pack);
285 // the scan hashed v0; serve a different zlib member of the same length there on every read
286 const altered = pack.slice();
287 const first = { at: idx.columns.offset[0] + idx.columns.headerBytes[0], len: idx.columns.compressed[0] };
288 const other = await zlibDeflate(te.encode(`version zerO ${'q'.repeat(60)}\n`));
289 assert.equal(other.length, first.len, 'same-length member');
290 altered.set(other, first.at);
291 const changed = { size: pack.length, read: (o, l) => (o === first.at ? rangeFromBytes(altered).read(o, l) : good.read(o, l)) };
292 const r = await resolvePackRange(idx, changed, { cacheBytes: 0 });
293 assert.equal(r.ok, false);
294 assert.equal(r.error.code, 'pack_corrupt');
295 assert.match(r.error.message, /read back as/);
296 // the range source's own refusal is passed through
297 const refusing = { size: pack.length, read: async () => ({ ok: false, error: Object.assign(new Error('gone'), { code: 'network' }) }) };
298 assert.equal((await resolvePackRange(idx, refusing)).error.code, 'network');
299 // and a well-behaved one resolves
300 assert.equal(must(await resolvePackRange(idx, good, { cacheBytes: 0 })).oid(2).value, hashObject('blob', v2));
301 void ok;
302});
304test('refusals at the read: work past the budget on a whole base, a member that ends early, a source that serves more, a thin base of a delta type or gone when asked again', async () => {
305 const v0 = te.encode(`base ${Array.from({ length: 60 }, (_, i) => (i * 7919) % 1009).join(',')}\n`);
306 const v1 = concat([v0, te.encode('one\n')]);
307 const pack = await packOf([{ type: TYPE.BLOB, data: v0 }, { data: editDelta(v0, v1, 20), baseIndex: 0 }]);
308 const idx = must(await scan(pack));
309 const good = rangeFromBytes(pack);
310 const first = { at: idx.columns.offset[0] + idx.columns.headerBytes[0], len: idx.columns.compressed[0] };
311 const serving = (at, bytes) => ({ size: pack.length, read: (o, l) => (o === at ? Promise.resolve(ok(new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } }))) : good.read(o, l)) });
312 // the whole base read back to apply the delta is, alone, past the work budget
313 const work = await resolvePackRange(idx, good, { limits: withLimits({ maxInflatedTotal: v0.length - 1 }), budget: RETAIN_BUDGET, cacheBytes: 0 });
314 assert.equal(work.error.code, 'limit_exceeded');
315 assert.equal(work.error.what, 'resolve work bytes');
316 // a zlib member that ends before the length the index gives (the rest of the range is junk)
317 const short = await zlibDeflate(new Uint8Array(v0.length));
318 assert.ok(short.length < first.len, 'a shorter member');
319 const padded = new Uint8Array(first.len);
320 padded.set(short);
321 const early = await resolvePackRange(idx, serving(first.at, padded), { cacheBytes: 0 });
322 assert.equal(early.error.code, 'pack_corrupt');
323 assert.match(early.error.message, /zlib member is \d+ of \d+ bytes read, the index says \d+/);
324 // a source that serves more than the range it was asked for
325 const more = await resolvePackRange(idx, { size: pack.length, read: (o, l) => good.read(o, o === first.at ? l + 3 : l) }, { cacheBytes: 0 });
326 assert.equal(more.error.code, 'pack_corrupt');
327 assert.match(more.error.message, /range source gave more than \d+ bytes/);
328 // b rests (REF) on a, a on z outside the pack: z handed back as a delta type, then z found once and not again
329 const z = te.encode(`thin ${'t'.repeat(40)}\n`);
330 const a = concat([z, te.encode('alpha\n')]);
331 const b = concat([a, te.encode('beta\n')]);
332 const [zo, ao] = [hashObject('blob', z), hashObject('blob', a)];
333 const thin = await packOf([{ data: editDelta(a, b, 10), baseOid: ao }, { data: editDelta(z, a, 10), baseOid: zo }]);
334 const typed = await resolved(thin, { bases: oid => (oid === zo ? { type: TYPE.OFS_DELTA, data: z } : null) });
335 assert.equal(typed.error.code, 'pack_corrupt');
336 assert.match(typed.error.message, /ref delta base [0-9a-f]{40} has type 6/);
337 // Two chains rest on z. The last result something rests on is pinned and never made again; the other is, with no
338 // cache, rebuilt from z, which is asked for a second time. (Before the pin, one chain was enough to ask twice.)
339 const a2 = concat([z, te.encode('gamma\n')]);
340 const b2 = concat([a2, te.encode('delta\n')]);
341 const forked = await packOf([{ data: editDelta(a, b, 10), baseOid: ao }, { data: editDelta(z, a, 10), baseOid: zo }, { data: editDelta(a2, b2, 10), baseOid: hashObject('blob', a2) }, { data: editDelta(z, a2, 10), baseOid: zo }]);
342 let answered = 0;
343 const gone = await resolved(forked, { cacheBytes: 0, bases: oid => (oid === zo && answered++ === 0 ? { type: TYPE.BLOB, data: z } : null) });
344 assert.equal(gone.error.code, 'pack_corrupt');
345 assert.match(gone.error.message, /not found when asked again/);
346});
348// --- the work bound (review of ec337c5): what a resolve costs, counted by the test's own RangeSource ---
349const TYPE_OF = { [TYPE.COMMIT]: 'commit', [TYPE.TREE]: 'tree', [TYPE.BLOB]: 'blob', [TYPE.TAG]: 'tag' };
350// A source that counts the range reads made for each entry (by the offset its zlib member starts at).
351function countingSource(pack, idx) {
352 const memberAt = new Map();
353 for (let i = 0; i < idx.columns.offset.length; i++) memberAt.set(idx.columns.offset[i] + idx.columns.headerBytes[i], i);
354 const reads = new Map();
355 const inner = rangeFromBytes(pack);
356 return { reads, source: { size: inner.size, read: (offset, length) => { const i = memberAt.get(offset); reads.set(i, (reads.get(i) ?? 0) + 1); return inner.read(offset, length); } } };
357}
359test('one large file edited 80 times (real git history) resolves on default options: every oid, type and size is git\'s, and the work is within 2x the sum of object sizes', async () => {
360 const dir = mkdtempSync(join(tmpdir(), 'ply-range-history-'));
361 try {
362 const git = (args, input) => gitIn(dir, ['-c', 'user.name=t', '-c', 'user.email=t@example.com', '-c', 'core.compression=1', ...args], input);
363 git(['init', '-q', '.']);
364 // a lockfile-like text file of about 4.3 MB; each commit changes one line
365 const lines = Array.from({ length: 36000 }, (_, i) => ` "pkg-${i}": { "version": "1.0.${i}", "integrity": "sha512-${((i * 2654435761) >>> 0).toString(16).padStart(8, '0').repeat(8)}" },`);
366 for (let c = 0; c < 80; c++) {
367 lines[(c * 449) % lines.length] = ` "edited-${c}": { "version": "2.0.${c}" },`;
368 writeFileSync(join(dir, 'package-lock.json'), lines.join('\n'));
369 git(['add', 'package-lock.json']);
370 git(['commit', '-q', '-m', `edit ${c}`]);
371 }
372 const pack = new Uint8Array(git(['pack-objects', '--all', '--stdout', '-q', '--delta-base-offset'], ''));
373 // git's own list of the repository's objects: oid, type, size
374 const want = git(['cat-file', '--batch-all-objects', '--batch-check']).toString().trim().split('\n');
375 const sizeOf = new Map(want.map(l => { const [oid, , size] = l.split(' '); return [oid, Number(size)]; }));
376 const sum = [...sizeOf.values()].reduce((a, b) => a + b, 0);
377 // the defect needs a history whose objects sum past the old flat allowance (256 MiB) in a pack 32x smaller
378 assert.ok(sum > 256 * 1024 * 1024 && pack.length * 32 < 256 * 1024 * 1024, `history ${sum} bytes in a pack of ${pack.length}`);
379 const idx = must(await scan(pack));
380 const { reads, source } = countingSource(pack, idx);
381 const res = must(await resolvePackRange(idx, source));
382 const got = Array.from({ length: res.count }, (_, i) => `${res.oid(i).value} ${TYPE_OF[res.type(i).value]} ${res.columns.size[i]}`);
383 assert.deepEqual(got.toSorted(), want.toSorted());
384 assertMatchesGit(pack, idx, res);
385 // work, counted here: each read inflates its member (size by node:zlib) and, for a delta, produces its object (size by git)
386 const byOffset = verifyPack(pack);
387 let work = 0;
388 for (const [i, times] of reads) {
389 const at = idx.columns.offset[i];
390 const member = inflateSync(pack.subarray(at + idx.columns.headerBytes[i], at + idx.columns.headerBytes[i] + idx.columns.compressed[i])).length;
391 const g = byOffset.get(at);
392 work += times * (member + (g.base === null ? 0 : sizeOf.get(g.oid)));
393 }
394 console.log(`large-file history: pack ${pack.length} B, ${res.count} objects, sum of sizes ${sum}, work ${work} (${(work / sum).toFixed(3)}x sum, ${(work / pack.length).toFixed(1)}x pack), reads ${[...reads.values()].reduce((a, b) => a + b, 0)}`);
395 assert.ok(work <= 2 * sum, `work ${work} is over twice the sum of object sizes ${sum}`);
396 } finally { rmSync(dir, { recursive: true, force: true }); }
397}, 600_000);
399test('a linear chain of objects larger than the cache: every entry is read once, not once per link above it', async () => {
400 const { entries, oids } = refChain(30);
401 const pack = await packOf(entries);
402 const idx = must(await scan(pack));
403 const { reads, source } = countingSource(pack, idx);
404 // each version is over 50 bytes, so none fits a 16-byte cache
405 const res = must(await resolvePackRange(idx, source, { cacheBytes: 16 }));
406 for (const oid of oids) assert.equal(res.find(oid).tag, 'some');
407 // by hand: 31 entries (the root and 30 deltas); the root is read to start the chain and each delta once, to be
408 // applied to the result made just before it
409 assert.equal([...reads.values()].reduce((a, b) => a + b, 0), 31);
410 assert.deepEqual([...new Set(reads.values())], [1]);
411});
413test('budget fields are checked where they enter: a ratio or work ceiling that is NaN, Infinity or negative is bad_request', async () => {
414 const pack = await packOf(refChain(3).entries);
415 for (const bad of [Number.NaN, Number.POSITIVE_INFINITY, -1]) {
416 const ratio = await resolved(pack, { budget: { tag: 'work', ratio: bad, capObjects: false } });
417 assert.equal(ratio.ok, false, `ratio ${bad} was accepted`);
418 assert.equal(ratio.error.code, 'bad_request');
419 const ceiling = await resolved(pack, { limits: { maxResolveWork: bad } });
420 assert.equal(ceiling.ok, false, `maxResolveWork ${bad} was accepted`);
421 assert.equal(ceiling.error.code, 'bad_request');
422 }
423 // the ceiling is hard: a chain that needs more work than it allows is refused. The ceiling is never below the
424 // record's maxInflatedTotal, so both are lowered: max(10, 10) = 10
425 const low = await resolved(pack, { limits: { maxResolveWork: 10, maxInflatedTotal: 10 } });
426 assert.equal(low.error.code, 'limit_exceeded');
427 assert.equal(low.error.what, 'resolve work bytes');
428});
430test('a range source that throws, rejects or errors its stream mid-read is a coded error, not a rejection', async () => {
431 const pack = await packOf(refChain(3).entries);
432 const idx = must(await scan(pack));
433 const inner = rangeFromBytes(pack);
434 const sources = {
435 rejects: { size: inner.size, read: () => Promise.reject(new Error('connection reset')) },
436 throws: { size: inner.size, read: () => { throw new Error('connection reset'); } },
437 'errors its stream': { size: inner.size, read: async () => ok(new ReadableStream({ pull(c) { c.error(new Error('connection reset')); } })) },
438 };
439 for (const [name, source] of Object.entries(sources)) {
440 const r = await resolvePackRange(idx, source);
441 assert.equal(r.ok, false, name);
442 assert.equal(r.error.code, 'network', name);
443 assert.match(r.error.message, /connection reset/, name);
444 }
445 // a thin-base lookup that throws is the same kind of failure
446 const thin = await packOf([{ data: editDelta(te.encode('x'.repeat(50)), te.encode('y'.repeat(50))), baseOid: 'aa'.repeat(20) }]);
447 const t = await resolved(thin, { bases: () => { throw new Error('store down'); } });
448 assert.equal(t.ok, false);
449 assert.equal(t.error.code, 'network');
450});
452// The Abort contract (README): an aborted resolve settles `aborted` when its signal fires, not when the storage answers.
453// git's OFS pack has a delta, so resolving it reads its entries back through the source; the source here never answers.
454// Expected values are the contract's: the code `aborted`, no read issued for a signal aborted before the call, and for
455// one aborted mid-read exactly the one read in flight, told through its signal (and its range cancelled once open).
456test('an aborted resolve settles aborted: before the call (no read), at the range open, and mid-read of a range', async () => {
457 const { packs } = buildPacks();
458 const bytes = packs.ofs.bytes;
459 const idx = must(await scan(bytes));
460 const before = stallAt(bytes, 0, 'never');
461 const pre = await resolvePackRange(idx, before, { signal: abortedBefore() });
462 assert.deepEqual({ code: codeOf(pre), ...before.state }, { code: 'aborted', reads: 0, cancelled: 0, aborted: 0 });
463 const seen = {};
464 for (const how of ['never', 'stall']) {
465 const source = stallAt(bytes, 0, how);
466 const ac = new AbortController();
467 const op = resolvePackRange(idx, source, { signal: ac.signal });
468 assert.equal(await settledWithin(source.parked), 'settled', how);
469 assert.equal(await settledWithin(op, 20), 'pending', how);
470 ac.abort(new Error('stop'));
471 assert.equal(await settledWithin(op), 'settled', how);
472 seen[how] = { code: codeOf(await op), ...source.state };
473 }
474 assert.deepEqual(seen, {
475 never: { code: 'aborted', reads: 1, cancelled: 0, aborted: 1 },
476 stall: { code: 'aborted', reads: 1, cancelled: 1, aborted: 1 },
477 });
478});
480test('the visit of each child is charged: one blob and one REF delta on it is refused at the visit when the ceiling is the blob (#299)', async () => {
481 // by hand, from the charge model at the top of src/packrangeresolve.ts: the blob inflated (6), then its one child visited
482 // (1) = 7 > 6, before the delta is inflated (4) or applied (6). Without the visit charge the refusal is at the delta's
483 // inflate, 6 + 4 + 1 = 11 with the visit counted after or 6 + 4 = 10 with none.
484 const data = te.encode('hello\n');
485 const pack = await packOf([{ type: TYPE.BLOB, data }, { data: Uint8Array.of(6, 6, 0x90, 6), baseOid: hashObject('blob', data) }]);
486 const r = await resolved(pack, { limits: withLimits({ maxInflatedTotal: 6, maxResolveWork: 6 }), budget: RETAIN_BUDGET });
487 assert.equal(r.ok, false);
488 assert.equal(r.error.code, 'limit_exceeded');
489 assert.equal(r.error.what, 'resolve work bytes');
490 assert.equal(r.error.got, 7);
491});
Diagnostics6 git calls · 87 ms
CallStatusSizeTime087 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=11990 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=78 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetch2009.7 KiB78 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=9956 git.response.cache=MISS git.response.duration_ms=78 git.fetch.wants=f12ee45d6590af148d6efc5e2cd54860a7b6787b git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=9956 git.pkt.payload.max=9929 git.sideband.band1.bytes=9929 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e1555c386a90ce
- Request
- 2724995a…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format