// scanPackIndex (#236): a pack read from a stream, indexed without resolving deltas or keeping objects. Held to // describePack (the buffered layout) field by field, to `git verify-pack -v` on packs git wrote (this repository's // objects, OFS and REF deltas), and to itself across every chunking (whole, one byte, random cuts) and both source // kinds (ReadableStream, AsyncIterable). Then the refusals: an OFS base off an entry boundary, truncation, a bad // trailer, corrupt zlib, a selected object over the cap, and the source cancelled on each. import { test } from 'vitest'; import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { Rng, splitRandom } from './fuzz/prng.js'; import { scanPackIndex, SCAN_INDEX_ENTRY_BYTES } from '../src/packscanindex.js'; import { describePack, encodeObjHeader, zlibDeflate } from '../src/pack.js'; import { hashObject, TYPE } from '../src/objects.js'; import { Sha1, fromHex } from '../src/sha1.js'; import { concat, te } from '../src/bytes.js'; import { withLimits } from '../src/limits.js'; import { PACK_SIGNATURE, PACK_VERSION, PACK_HEADER_BYTES, UINT32_BYTES, OBJ_CONTINUE, OBJ_SIZE_NEXT_MASK, OBJ_SIZE_NEXT_BASE } from '../src/format.js'; import { buildPacks } from './space/pack-fixtures.js'; import { settledWithin } from './abortcells.js'; const must = r => { if (r.ok) return r.value; throw Object.assign(new Error(r.error.message), { code: r.error.code }); }; function encodeOfs(off) { const out = [off & OBJ_SIZE_NEXT_MASK]; while ((off = Math.floor(off / OBJ_SIZE_NEXT_BASE)) > 0) out.unshift(OBJ_CONTINUE | (--off & OBJ_SIZE_NEXT_MASK)); return Uint8Array.of(...out); } // entries: { type, data } whole, { data, baseOid } REF_DELTA, { data, baseIndex } OFS_DELTA to an earlier entry, // { data, ofsDistance } an OFS_DELTA with a raw distance (to aim it off an entry boundary). async function packOf(entries) { const parts = [Uint8Array.of(...PACK_SIGNATURE, 0, 0, 0, PACK_VERSION), new Uint8Array(UINT32_BYTES)]; new DataView(parts[1].buffer).setUint32(0, entries.length); const offsets = []; let at = PACK_HEADER_BYTES; for (const e of entries) { offsets.push(at); const ofs = e.baseIndex !== undefined ? at - offsets[e.baseIndex] : e.ofsDistance; const type = e.baseOid ? TYPE.REF_DELTA : ofs !== undefined ? TYPE.OFS_DELTA : e.type; const entry = concat([ encodeObjHeader(type, e.data.length), ...(e.baseOid ? [fromHex(e.baseOid)] : ofs !== undefined ? [encodeOfs(ofs)] : []), await zlibDeflate(e.data), ]); parts.push(entry); at += entry.length; } const body = concat(parts); return concat([body, new Sha1().update(body).digest()]); } // A delta from base to target: copy nothing, insert the target (valid instructions; the scan never applies them). const insertDelta = (base, target) => { const varint = n => { const out = []; do { let b = n & 0x7f; n >>>= 7; if (n) b |= 0x80; out.push(b); } while (n); return out; }; const ops = []; for (let i = 0; i < target.length; i += 127) { const piece = target.subarray(i, i + 127); ops.push(piece.length, ...piece); } return Uint8Array.of(...varint(base.length), ...varint(target.length), ...ops); }; // Sources over the same chunks. const streamOf = (chunks, onCancel = () => {}) => { let i = 0; return new ReadableStream({ pull(c) { if (i < chunks.length) c.enqueue(chunks[i++]); else c.close(); }, cancel() { onCancel(); } }); }; async function* iterOf(chunks) { for (const c of chunks) yield c; } const feedings = (bytes, rng) => [[bytes], Array.from(bytes, b => Uint8Array.of(b)), splitRandom(rng, bytes, { allowEmpty: false })]; const plain = e => ({ ...e }); const rows = idx => [...idx.entries()].map(plain); // The fields describePack and the scan share: the layout, and the resolved type and oid where the scan can know them. function assertMatchesLayout(idx, layout) { assert.equal(idx.count, layout.count); assert.equal(idx.trailer, layout.trailer); assert.equal(idx.columns.offset.length, layout.objects.length); for (const [i, o] of layout.objects.entries()) { const found = idx.entry(i); assert.equal(found.tag, 'some', `entry ${i}`); const e = found.value; assert.equal(e.index, i); assert.equal(e.offset, o.offset, `offset of ${i}`); assert.equal(e.headerBytes, o.headerBytes, `headerBytes of ${i}`); assert.equal(e.compressed, o.compressed, `compressed of ${i}`); assert.equal(e.size, o.size, `size of ${i}`); assert.equal(e.storedType, o.wireType, `stored type of ${i}`); assert.equal(e.base.tag, o.base.tag, `base of ${i}`); if (o.base.tag === 'ofs') { assert.equal(e.base.offset, o.base.offset); assert.equal(idx.columns.offset[e.base.index], o.base.offset); } if (o.base.tag === 'ref') assert.equal(e.base.oid, o.base.oid); if (e.type !== null) assert.equal(e.type, o.type, `resolved type of ${i}`); if (e.oid !== null) assert.equal(e.oid, o.oid, `oid of ${i}`); if (o.base.tag === 'none') { assert.equal(e.type, o.wireType); assert.equal(e.base.tag, 'none'); } } } test('a pack of every whole type and an OFS chain matches describePack, for every chunking and both source kinds', async () => { const base = te.encode('the quick brown fox '.repeat(40)); const v2 = te.encode('the quick brown fox jumps '.repeat(40)); const v3 = te.encode('the quick brown fox jumps over '.repeat(40)); const pack = await packOf([ { type: TYPE.COMMIT, data: te.encode('tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\nauthor A 0 +0000\ncommitter A 0 +0000\n\nm\n') }, { type: TYPE.TREE, data: new Uint8Array(0) }, { type: TYPE.BLOB, data: base }, { type: TYPE.TAG, data: te.encode('object 4b825dc642cb6eb9a060e54bf8d69288fbee4904\ntype tree\ntag t\ntagger A 0 +0000\n\nt\n') }, { data: insertDelta(base, v2), baseIndex: 2 }, { data: insertDelta(v2, v3), baseIndex: 4 }, ]); const layout = must(describePack(pack)); const rng = new Rng(236); let reference = null; for (let round = 0; round < 8; round++) { for (const chunks of feedings(pack, rng)) { for (const source of [streamOf(chunks), iterOf(chunks)]) { const idx = must(await scanPackIndex(source, { oids: true })); assertMatchesLayout(idx, layout); assert.deepEqual([...idx.columns.type], [TYPE.COMMIT, TYPE.TREE, TYPE.BLOB, TYPE.TAG, TYPE.BLOB, TYPE.BLOB]); assert.equal(idx.bytes, pack.length); if (reference === null) reference = rows(idx); else assert.deepEqual(rows(idx), reference); } } } }); test('REF_DELTA: a whole base earlier or later in the pack resolves with oids on; absent, a delta base, or oids off is unknown', async () => { const a = te.encode('alpha '.repeat(30)); const b = te.encode('alpha beta '.repeat(30)); const c = te.encode('alpha beta gamma '.repeat(30)); const aOid = hashObject('blob', a); const bOid = hashObject('blob', b); const absent = 'ab'.repeat(20); const pack = await packOf([ { data: insertDelta(a, b), baseOid: aOid }, // 0: base later (entry 1) { type: TYPE.BLOB, data: a }, // 1: whole { data: insertDelta(a, c), baseOid: aOid }, // 2: base earlier { data: insertDelta(a, te.encode('three')), baseOid: absent }, // 3: base absent (thin) { data: insertDelta(b, te.encode('four')), baseOid: bOid }, // 4: base is entry 0, a delta: unknown without resolving // distinct results: describePack keys resolved objects by oid, and git never packs one object twice { data: insertDelta(c, te.encode('five')), baseIndex: 2 }, // 5: OFS onto a resolved REF delta { data: insertDelta(c, te.encode('six')), baseIndex: 3 }, // 6: OFS onto an unknown one ]); const on = must(await scanPackIndex(streamOf([pack]), { oids: true })); assert.deepEqual([...on.columns.type], [TYPE.BLOB, TYPE.BLOB, TYPE.BLOB, 0, 0, TYPE.BLOB, 0]); assert.deepEqual([...on.columns.baseIndex], [1, -1, 1, -1, -1, 2, 3]); const e0 = on.entry(0); assert.equal(e0.tag, 'some'); assert.deepEqual(e0.value.base, { tag: 'ref', oid: aOid, index: 1 }); assert.equal(e0.value.oid, null); assert.equal(on.entry(1).value.oid, aOid); assert.equal(on.entry(3).value.type, null); assert.equal(on.heldBytes, 7 * (SCAN_INDEX_ENTRY_BYTES + 20 + 20)); const off = must(await scanPackIndex(streamOf([pack]), { oids: false })); assert.deepEqual([...off.columns.type], [0, TYPE.BLOB, 0, 0, 0, 0, 0]); assert.equal(off.entry(1).value.oid, null); assert.equal(off.heldBytes, 7 * (SCAN_INDEX_ENTRY_BYTES + 20)); assertMatchesLayout(on, must(describePack(pack))); assert.equal(on.entry(7).tag, 'none'); assert.equal(on.entry(-1).tag, 'none'); assert.equal(on.entry(0.5).tag, 'none'); }); // git's own packs: the fixtures (every type, OFS, REF, an OFS chain, a thin pack) and this repository's objects, with // and without --delta-base-offset, against describePack and `git verify-pack -v`. const gitIn = (cwd, args, input) => execFileSync('git', args, { cwd, input, maxBuffer: 1 << 30 }); function verifyPack(pack) { const dir = mkdtempSync(join(tmpdir(), 'ply-scan-index-')); try { const file = join(dir, 'p.pack'); writeFileSync(file, pack); gitIn(dir, ['index-pack', file]); const out = gitIn(dir, ['verify-pack', '-v', join(dir, 'p.idx')]).toString(); const byOffset = new Map(); for (const line of out.split('\n')) { const m = /^([0-9a-f]{40}) (commit|tree|blob|tag)\s+(\d+) (\d+) (\d+)/.exec(line); if (m) byOffset.set(Number(m[5]), { oid: m[1], type: TYPE[m[2].toUpperCase()], inPack: Number(m[4]) }); } return byOffset; } finally { rmSync(dir, { recursive: true, force: true }); } } async function againstGit(pack) { const idx = must(await scanPackIndex(iterOf(splitRandom(new Rng(pack.length), pack, { allowEmpty: false })), { oids: true })); assertMatchesLayout(idx, must(describePack(pack))); const git = verifyPack(pack); assert.equal(git.size, idx.count); let known = 0; for (const e of idx.entries()) { const g = git.get(e.offset); assert.ok(g, `git has no object at ${e.offset}`); assert.equal(e.headerBytes + e.compressed, g.inPack, `size in pack at ${e.offset}`); if (e.oid !== null) assert.equal(e.oid, g.oid); if (e.type !== null) { assert.equal(e.type, g.type, `type at ${e.offset}`); known += 1; } } return { idx, known }; } test('git-written fixture packs: every type, OFS, REF, an OFS chain, match describePack and git verify-pack', async () => { const { packs } = buildPacks(); for (const name of ['full', 'ofs', 'ref', 'chain']) { const { idx, known } = await againstGit(packs[name].bytes); assert.equal(known, idx.count, `${name}: every type is known`); } // a thin pack: its REF bases are outside it, so those types are unknown, never guessed const thin = must(await scanPackIndex(streamOf([packs.thin.bytes]), { oids: true })); assertMatchesLayout(thin, must(describePack(packs.thin.bytes))); for (const e of thin.entries()) if (e.base.tag === 'ref' && e.base.index === -1) assert.equal(e.type, null); }); test("this repository's objects, packed by git with OFS and with REF deltas, match describePack and git verify-pack", async () => { const objects = gitIn(process.cwd(), ['rev-list', '--objects', 'HEAD', '-n', '400']); for (const args of [['--delta-base-offset'], []]) { const pack = new Uint8Array(gitIn(process.cwd(), ['pack-objects', '--stdout', '-q', ...args], objects)); const { idx, known } = await againstGit(pack); // CI checks out shallow: a few hundred objects there, thousands in a full clone assert.ok(idx.count > 100, `only ${idx.count} objects`); // OFS chains rest on whole objects in the pack: all known. REF chains know every type whose base is whole. if (args.length > 0) assert.equal(known, idx.count); else assert.ok(known > 0); } }); test('a REF_DELTA-only pack (what Artifacts sends) resolves every depth-1 delta by default, at 71 bytes per entry', async () => { const objects = gitIn(process.cwd(), ['rev-list', '--objects', 'HEAD', '-n', '400']); const pack = new Uint8Array(gitIn(process.cwd(), ['pack-objects', '--stdout', '-q', '--depth=1', '--no-reuse-delta'], objects)); const idx = must(await scanPackIndex(iterOf(splitRandom(new Rng(7), pack, { allowEmpty: false })))); const stored = [...idx.columns.storedType]; assert.equal(stored.includes(TYPE.OFS_DELTA), false); const refs = stored.filter(t => t === TYPE.REF_DELTA).length; assert.ok(refs > 0, 'git wrote no REF deltas'); // depth 1: every base is a whole object in the pack, so every type is known assert.equal([...idx.columns.type].filter(t => t === 0).length, 0); assert.equal(idx.heldBytes, idx.count * (SCAN_INDEX_ENTRY_BYTES + 20 + 20)); const git = verifyPack(pack); for (const e of idx.entries()) assert.equal(e.type, git.get(e.offset).type); }); test('selected content: whole objects the caller picks, in pack order, with their oid; others are not buffered', async () => { const { packs } = buildPacks(); const seen = []; const idx = must(await scanPackIndex(streamOf([packs.full.bytes]), { oids: true, content: { select: h => h.type === TYPE.COMMIT || h.type === TYPE.TAG, take: async (o, data) => { await Promise.resolve(); seen.push({ ...o, data: new TextDecoder().decode(data) }); }, }, })); assert.deepEqual(seen.map(s => s.type), [TYPE.COMMIT, TYPE.TAG]); for (const s of seen) { const e = idx.entry(s.index).value; assert.equal(s.oid, e.oid); assert.equal(hashObject(s.type === TYPE.COMMIT ? 'commit' : 'tag', te.encode(s.data)), s.oid); assert.equal(s.size, e.size); } // over maxObjectSize: refused before it is buffered, the source cancelled let cancelled = 0; const big = must(describePack(packs.full.bytes)).objects.reduce((m, o) => Math.max(m, o.size), 0); const r = await scanPackIndex(streamOf(splitRandom(new Rng(3), packs.full.bytes, { allowEmpty: false }), () => { cancelled += 1; }), { limits: withLimits({ maxObjectSize: big - 1 }), content: { select: () => true, take: () => {} }, }); assert.equal(r.ok, false); assert.equal(r.error.code, 'limit_exceeded'); assert.equal(r.error.what, 'selected object bytes'); assert.equal(cancelled, 1); }); test('refusals: OFS base off an entry boundary, truncation at every cut, a bad trailer, corrupt zlib; each cancels the source', async () => { const a = te.encode('alpha '.repeat(30)); const good = await packOf([{ type: TYPE.BLOB, data: a }, { type: TYPE.BLOB, data: te.encode('b') }]); // the second object's OFS distance lands one byte into the first entry const off = await packOf([{ type: TYPE.BLOB, data: a }, { data: insertDelta(a, a), ofsDistance: 0 }]); const firstLen = must(describePack(good)).objects[0].headerBytes + must(describePack(good)).objects[0].compressed; const offBoundary = await packOf([{ type: TYPE.BLOB, data: a }, { data: insertDelta(a, a), ofsDistance: firstLen - 1 }]); const run = async bytes => scanPackIndex(streamOf([bytes])); const r1 = await run(offBoundary); assert.equal(r1.ok, false); assert.equal(r1.error.code, 'pack_corrupt'); assert.match(r1.error.message, /ofs delta base not at object boundary 13/); assert.equal((await run(off)).error.code, 'pack_corrupt'); // every truncation of a good pack is pack_corrupt for (let cut = 0; cut < good.length; cut++) { const r = await run(good.subarray(0, cut)); assert.equal(r.ok, false, `cut ${cut}`); assert.equal(r.error.code, 'pack_corrupt', `cut ${cut}`); } const badTrailer = good.slice(); badTrailer[badTrailer.length - 1] ^= 1; assert.match((await run(badTrailer)).error.message, /trailer sha mismatch/); // verify off: the trailer is reported, not checked assert.equal(must(await scanPackIndex(streamOf([badTrailer]), { verify: false })).count, 2); const badZlib = good.slice(); badZlib[PACK_HEADER_BYTES + 3] ^= 0xff; assert.equal((await run(badZlib)).error.code, 'pack_corrupt'); // a header that claims more objects than arrive const lying = good.slice(); new DataView(lying.buffer).setUint32(8, 3); assert.equal((await run(lying)).error.code, 'pack_corrupt'); // a refusal before the input ends cancels the source: the rest of it is never read let cancelled = 0; let pulled = 0; const oneByte = Array.from(offBoundary, b => Uint8Array.of(b)); const counting = new ReadableStream({ pull(c) { if (pulled < oneByte.length) c.enqueue(oneByte[pulled++]); else c.close(); }, cancel() { cancelled += 1; } }); assert.equal((await scanPackIndex(counting)).error.code, 'pack_corrupt'); assert.equal(cancelled, 1); assert.ok(pulled < oneByte.length, `read ${pulled} of ${oneByte.length} bytes`); // an AsyncIterable source is closed (return()) on a refusal let closed = false; async function* gen() { try { yield offBoundary; } finally { closed = true; } } assert.equal((await scanPackIndex(gen())).ok, false); assert.equal(closed, true); // limits a caller passes are checked before anything is read const badLimits = await scanPackIndex(streamOf([good]), { limits: { maxPackObjects: -1 } }); assert.equal(badLimits.ok, false); }); test('maxPackObjects refuses at the header; the columns never exceed the count that arrived', async () => { const pack = await packOf(Array.from({ length: 3000 }, (_, i) => ({ type: TYPE.BLOB, data: te.encode(`${i}`) }))); const r = await scanPackIndex(streamOf([pack]), { limits: withLimits({ maxPackObjects: 2999 }) }); assert.equal(r.error.code, 'limit_exceeded'); // the options of one call never leak into the next: the defaults are a fresh record each time const idx = must(await scanPackIndex(streamOf([pack]))); // oids on by default: 31 bytes of columns and a 20-byte oid per entry assert.equal(idx.heldBytes, 3000 * (SCAN_INDEX_ENTRY_BYTES + 20)); assert.equal(idx.columns.size.length, 3000); }); // The Abort contract (README): an aborted scan settles `aborted` when its signal fires, whichever of the caller's // awaits it is parked on: its source (a stream that served the pack header and then never another byte) or its content // sink (a take() that never settles). Expected values are the contract's: the code `aborted`, the stream cancelled // once, and take() called once (the first selected object) and never again. test('an aborted scan settles aborted while its source stalls mid-pack and while content.take never settles', async () => { const { packs } = buildPacks(); const bytes = packs.full.bytes; let cancelled = 0, reached; let parked = new Promise(r => { reached = r; }); let served = false; const stalled = new ReadableStream({ pull(c) { if (!served) { served = true; c.enqueue(bytes.subarray(0, 12)); return undefined; } reached(); return new Promise(() => {}); }, cancel() { cancelled++; }, }, { highWaterMark: 0 }); const a = new AbortController(); const scanA = scanPackIndex(stalled, { signal: a.signal }); await parked; assert.equal(await settledWithin(scanA, 20), 'pending', 'parked on the source'); a.abort(new Error('stop')); assert.equal(await settledWithin(scanA), 'settled', 'the source stalls'); const ra = await scanA; assert.deepEqual({ code: ra.ok ? 'ok' : ra.error.code, cancelled }, { code: 'aborted', cancelled: 1 }); let takes = 0; parked = new Promise(r => { reached = r; }); const b = new AbortController(); const scanB = scanPackIndex(streamOf([bytes]), { signal: b.signal, content: { select: () => true, take: () => { takes++; reached(); return new Promise(() => {}); } } }); await parked; assert.equal(await settledWithin(scanB, 20), 'pending', 'parked on take'); b.abort(new Error('stop')); assert.equal(await settledWithin(scanB), 'settled', 'take never settles'); const rb = await scanB; assert.deepEqual({ code: rb.ok ? 'ok' : rb.error.code, takes }, { code: 'aborted', takes: 1 }); });