// The async-iterable adapters over the pull decoders (#241, src/pullstream.ts): scanPack, pktFrames, inflateChunks.
// - identical items to the pull API over every chunking (whole, one byte at a time, seeded random cuts), from a
// ReadableStream and from an AsyncIterable
// - a break, a throw in the loop body and reader.cancel() cancel the source (cancel(reason) / return() observed)
// - a refusal errors the stream with the decoder's typed PlyError and cancels the source; malformed input of every
// kind each decoder refuses; a source that errors errors the stream with its own error; a chunk that is not bytes
// - backpressure: the source is read one chunk per need and never ahead; a consumer that stops stops the reads
// Memory (the adapters hold the decoders' bound) is measured in test/stream-memory.test.js (scan-iter, inflate-iter).
import { test } from 'vitest';
import assert from 'node:assert/strict';
import zlib from 'node:zlib';
import { Rng, splitRandom } from './fuzz/prng.js';
import { concat, te } from '../src/bytes.js';
import { pkt, pktInit, pktStep, pktFinish, FLUSH, DELIM } from '../src/pktline.js';
import { buildPack, makeDelta } from '../src/pack.js';
import { inflateInit, inflateFeed, inflatePull, inflateEnd, cappedBound, declaredBound, zlibWrap, rawWrap } from '../src/inflatestream.js';
import { parsePackInit, parsePackFeed, parsePackPull, parsePackEnd, WORK_BUDGET } from '../src/packstream.js';
import { scanPack, pktFrames, inflateChunks } from '../src/pullstream.js';
import { isPlyError } from '../src/errors.js';
import { hashObject, TYPE } from '../src/objects.js';
import { withLimits } from '../src/limits.js';
import { unwrap } from './unwrap.js';
// ---- sources
// A ReadableStream over `chunks` that records every pull and its cancel; highWaterMark 0, so a pull is a read.
function recorded(chunks) {
const log = { pulls: 0, cancelled: [], closed: false };
let i = 0;
const stream = new ReadableStream({
pull(c) { log.pulls += 1; if (i < chunks.length) c.enqueue(chunks[i++]); else { log.closed = true; c.close(); } },
cancel(why) { log.cancelled.push(why); },
}, { highWaterMark: 0 });
return { stream, log };
}
// An AsyncIterable over `chunks` that records next() and return().
function iterable(chunks) {
const log = { nexts: 0, returned: 0 };
return {
log,
source: {
[Symbol.asyncIterator]() {
let i = 0;
return {
async next() { log.nexts += 1; return i < chunks.length ? { done: false, value: chunks[i++] } : { done: true, value: undefined }; },
async return() { log.returned += 1; return { done: true, value: undefined }; },
};
},
},
};
}
const chunkings = bytes => {
const out = [['whole', [bytes]], ['bytes', Array.from(bytes, b => Uint8Array.of(b))]];
for (const seed of [1, 2, 3, 20261110]) out.push([`random ${seed}`, splitRandom(new Rng(seed), bytes, { maxChunks: 64 })]);
return out;
};
async function collect(stream) { const out = []; for await (const x of stream) out.push(x); return out; }
async function failure(stream) {
const seen = [];
try { for await (const x of stream) seen.push(x); } catch (e) { return { e, seen }; }
return assert.fail(`no error after ${seen.length} items`);
}
const ply = (e, code) => { assert.ok(isPlyError(e), `not a PlyError: ${e}`); assert.equal(e.code, code, e.message); };
const copy = p => (p.kind === 'chunk' ? { ...p, bytes: Uint8Array.from(p.bytes) } : p);
// ---- the reference drivers: the pull API, by hand
function pullPack(chunks, opts) {
const st = unwrap(parsePackInit(opts));
const out = [];
const drain = p => { for (;;) { const r = p ?? parsePackPull(st); p = undefined; if (!r.ok) return r.error; if (r.value.kind === 'need') return null; if (r.value.kind === 'end') return 'end'; out.push(copy(r.value)); } };
for (const c of chunks) { parsePackFeed(st, c); const d = drain(); if (d !== null) return { out, end: d }; }
return { out, end: drain(parsePackEnd(st)) };
}
function pullPkt(chunks) {
let st = pktInit();
const out = [];
for (const c of chunks) { const r = pktStep(st, c); if (!r.ok) return { out, end: r.error }; st = r.value.state; out.push(...r.value.frames); }
const f = pktFinish(st);
return { out, end: f.ok ? 'end' : f.error };
}
// ---- a pack with every kind of piece: whole objects, an OFS and a REF delta, a large blob of several chunks
const blob = n => Uint8Array.from({ length: n }, (_, i) => (i * 7 + (i >> 8)) & 255);
async function samplePack() {
const base = te.encode('line\n'.repeat(400));
const next = te.encode('line\n'.repeat(399) + 'changed\n');
const big = blob(80_000);
return (await buildPack([
{ type: TYPE.BLOB, data: base },
{ type: TYPE.BLOB, data: makeDelta(base, next).delta, base: { tag: 'ofs', index: 0 } },
{ type: TYPE.BLOB, data: big },
{ type: TYPE.BLOB, data: makeDelta(base, te.encode('x')).delta, base: { tag: 'ref', oid: hashObject('blob', base) } },
{ type: TYPE.COMMIT, data: te.encode('tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\nauthor a 1 +0000\ncommitter a 1 +0000\n\nm\n') },
])).pack;
}
test('scanPack: the same pieces as the pull API over every chunking, from a ReadableStream and an AsyncIterable', async () => {
const pack = await samplePack();
for (const [name, chunks] of chunkings(pack)) {
const want = pullPack(chunks, { verify: true, limits: withLimits({}) });
assert.equal(want.end, 'end', name);
const kinds = new Set(want.out.map(p => p.kind));
assert.deepEqual([...kinds].sort(), ['chunk', 'header', 'object', 'object-end', 'trailer'], name);
const fromStream = (await collect(scanPack(recorded(chunks).stream))).map(copy);
const fromIter = (await collect(scanPack(iterable(chunks).source))).map(copy);
assert.deepEqual(fromStream, want.out, `${name}: stream`);
assert.deepEqual(fromIter, want.out, `${name}: iterable`);
assert.ok(!fromStream.some(p => p.kind === 'need' || p.kind === 'end'), 'need and end are never items');
}
});
test('scanPack: options reach the scan (WORK_BUDGET reads a blob past maxObjectSize; verify off skips the trailer check)', async () => {
const pack = await samplePack();
const tight = withLimits({ maxObjectSize: 1024 });
ply((await failure(scanPack(recorded([pack]).stream, { limits: tight }))).e, 'limit_exceeded');
const worked = await collect(scanPack(recorded([pack]).stream, { limits: tight, budget: WORK_BUDGET }));
assert.equal(worked.filter(p => p.kind === 'object-end').length, 5);
const bad = Uint8Array.from(pack); bad[bad.length - 1] ^= 1;
ply((await failure(scanPack(recorded([bad]).stream))).e, 'pack_corrupt');
assert.equal((await collect(scanPack(recorded([bad]).stream, { verify: false }))).at(-1).kind, 'trailer');
});
test('scanPack: every refusal is the pull API\'s, typed, after the pieces before it; the source is cancelled with it', async () => {
const pack = await samplePack();
const flip = (at) => { const b = Uint8Array.from(pack); b[at] ^= 0xff; return b; };
const cases = [
['bad signature', flip(0), 'pack_corrupt'],
['bad version', flip(7), 'pack_corrupt'],
['truncated', pack.subarray(0, pack.length - 30), 'pack_corrupt'],
['trailer', flip(pack.length - 1), 'pack_corrupt'],
['corrupt member', flip(40), 'pack_corrupt'],
['empty', new Uint8Array(0), 'pack_corrupt'],
['too many objects', pack, 'limit_exceeded', withLimits({ maxPackObjects: 2 })],
['inflated total', pack, 'limit_exceeded', withLimits({ maxInflatedTotal: 10_000 })],
];
for (const [name, bytes, code, limits] of cases) {
for (const [cut, chunks] of chunkings(bytes).slice(0, 3)) {
const want = pullPack(chunks, { verify: true, limits: limits ?? withLimits({}) });
assert.notEqual(want.end, 'end', `${name} ${cut}: the pull API refuses`);
const { stream, log } = recorded(chunks);
const { e, seen } = await failure(scanPack(stream, limits ? { limits } : {}));
ply(e, code);
assert.equal(e.message, want.end.message, `${name} ${cut}`);
assert.deepEqual(seen.map(copy), want.out, `${name} ${cut}: the pieces before the refusal`);
if (!log.closed) assert.deepEqual(log.cancelled, [e], `${name} ${cut}: the source is cancelled with the refusal`);
}
}
});
test('scanPack: bad options error the stream on the first read and cancel the source', async () => {
const { stream, log } = recorded([new Uint8Array(1)]);
const { e } = await failure(scanPack(stream, { limits: { maxPackObjects: -1 } }));
ply(e, 'bad_request');
assert.deepEqual(log.cancelled, [e]);
assert.equal(log.pulls, 0, 'nothing was read');
const { e: e2 } = await failure(scanPack(recorded([]).stream, { budget: { tag: 'work', ratio: Number.NaN, capObjects: false } }));
ply(e2, 'bad_request');
});
test('a source that errors errors the stream with its own error (all three adapters, both source kinds)', async () => {
const boom = new Error('source died');
const pack = await samplePack();
const make = first => [
() => new ReadableStream({ start(c) { c.enqueue(first); }, pull(c) { c.error(boom); } }),
() => ({ [Symbol.asyncIterator]() { let n = 0; return { async next() { if (n++ === 0) return { done: false, value: first }; throw boom; } }; } }),
];
for (const [adapter, first] of [[scanPack, pack.subarray(0, 40)], [pktFrames, unwrap(pkt('a\n'))], [inflateChunks, zlib.deflateSync(blob(100)).subarray(0, 10)]]) {
for (const src of make(first)) {
const { e } = await failure(adapter(src()));
assert.equal(e, boom, `${adapter.name}: the source's own error`);
}
}
});
test('a chunk that is not a Uint8Array is bad_request (string, array, ArrayBuffer), and the source is cancelled', async () => {
for (const adapter of [scanPack, pktFrames, inflateChunks]) {
for (const junk of ['PACK', [1, 2, 3], new ArrayBuffer(4), 7]) {
const { stream, log } = recorded([junk]);
const { e } = await failure(adapter(stream));
ply(e, 'bad_request');
assert.match(e.message, /not a Uint8Array/);
assert.deepEqual(log.cancelled, [e], `${adapter.name} ${typeof junk}`);
}
}
});
test('break, a throw in the loop body, and reader.cancel() cancel the source; iteration after that is over', async () => {
const pack = await samplePack();
const frames = concat([unwrap(pkt('a\n')), unwrap(pkt('b\n')), FLUSH]);
const member = zlib.deflateSync(blob(200_000));
for (const [adapter, bytes] of [[scanPack, pack], [pktFrames, frames], [inflateChunks, member]]) {
const chunks = splitRandom(new Rng(9), bytes, { maxChunks: 16, allowEmpty: false });
// break after the first item
{ const { stream, log } = recorded(chunks); let n = 0; for await (const _ of adapter(stream)) { n++; break; } assert.equal(n, 1); assert.equal(log.cancelled.length, 1, `${adapter.name}: break`); assert.ok(log.pulls < chunks.length + 1, 'not read to the end'); }
// a throw in the body
{ const { stream, log } = recorded(chunks); const oops = new Error('body'); await assert.rejects((async () => { for await (const _ of adapter(stream)) throw oops; })(), e => e === oops); assert.equal(log.cancelled.length, 1, `${adapter.name}: throw`); }
// AsyncIterable source: return() is called
{ const { source, log } = iterable(chunks); for await (const _ of adapter(source)) break; assert.equal(log.returned, 1, `${adapter.name}: iterable return()`); }
// reader.cancel(reason) passes the reason through, and the stream reads done after it
{ const { stream, log } = recorded(chunks); const s = adapter(stream); const r = s.getReader(); assert.equal((await r.read()).done, false); const why = new Error('stop'); await r.cancel(why); assert.deepEqual(log.cancelled, [why]); assert.equal((await r.read()).done, true); }
// cancel before any read: the source is cancelled without being read
{ const { stream, log } = recorded(chunks); await adapter(stream).cancel('early'); assert.deepEqual(log.cancelled, ['early']); assert.equal(log.pulls, 0); }
}
});
test('a cancel while a read of the source is out: the read\'s chunk is not decoded and nothing is enqueued after', async () => {
let release;
const gate = new Promise(r => { release = r; });
let cancelled = 0;
const stream = new ReadableStream({ async pull(c) { await gate; c.enqueue(unwrap(pkt('late\n'))); }, cancel() { cancelled++; } }, { highWaterMark: 0 });
const r = pktFrames(stream).getReader();
const pending = r.read();
await new Promise(res => setTimeout(res, 10));
await r.cancel('stop');
release();
assert.equal((await pending).done, true);
assert.equal(cancelled, 1);
});
test('backpressure: one source read per need, none ahead; a consumer that stops reading stops the reads', async () => {
const pack = await samplePack();
const bytes1 = Array.from(pack, b => Uint8Array.of(b));
const { stream, log } = recorded(bytes1);
const r = scanPack(stream).getReader();
const header = await r.read();
assert.equal(header.value.kind, 'header');
// the scan decides the header on 32 bytes (a header and a trailer's worth, as parsePack): exactly that many reads
assert.equal(log.pulls, 32);
await new Promise(res => setTimeout(res, 30));
assert.equal(log.pulls, 32, 'no read while nobody reads');
const object = await r.read();
assert.equal(object.value.kind, 'object');
assert.ok(log.pulls <= 34, `the first object header: ${log.pulls} reads`);
// pkt-lines: a frame read a byte at a time is read to its last byte and no further
const two = concat([unwrap(pkt('x'.repeat(1000))), unwrap(pkt('y'))]);
const p = recorded(Array.from(two, b => Uint8Array.of(b)));
const pr = pktFrames(p.stream).getReader();
assert.equal((await pr.read()).value.payload.length, 1000);
assert.equal(p.log.pulls, 1004);
await pr.cancel();
// inflate: a 1 MiB member fed in 4 KiB chunks, the consumer takes one chunk: the reads stop at what one chunk needed
const member = zlib.deflateSync(blob(1 << 20), { level: 0 });
const m = recorded(Array.from({ length: Math.ceil(member.length / 4096) }, (_, i) => member.subarray(i * 4096, (i + 1) * 4096)));
const mr = inflateChunks(m.stream).getReader();
await mr.read();
assert.ok(m.log.pulls <= 6, `one 16 KiB chunk out: ${m.log.pulls} 4 KiB reads`);
await mr.cancel();
});
test('a byte at a time across a 65,516-byte pkt-line: no per-need hold (the reads run as fresh rounds)', async () => {
const big = unwrap(pkt(new Uint8Array(65_516).fill(97)));
const frames = await collect(pktFrames(recorded([...Array.from(big, b => Uint8Array.of(b)), FLUSH]).stream));
assert.deepEqual(frames.map(f => f.kind), ['data', 'flush']);
assert.equal(frames[0].payload.length, 65_516);
});
test('pktFrames: the same frames as pktStep over every chunking; data, flush, delim, response-end', async () => {
const wire = concat([unwrap(pkt('version 2\n')), DELIM, unwrap(pkt(new Uint8Array(3000).fill(1))), FLUSH, te.encode('0002'), unwrap(pkt('done'))]);
for (const [name, chunks] of chunkings(wire)) {
const want = pullPkt(chunks);
assert.equal(want.end, 'end');
assert.deepEqual(await collect(pktFrames(recorded(chunks).stream)), want.out, name);
assert.deepEqual(await collect(pktFrames(iterable(chunks).source)), want.out, name);
}
});
test('pktFrames: malformed input is pktStep\'s protocol refusal (bad hex, 0003, past LARGE_PACKET_MAX, cut inside a frame)', async () => {
const good = unwrap(pkt('ok\n'));
const cases = [
['bad hex', concat([good, te.encode('00zz')])],
['0003', concat([good, te.encode('0003')])],
['too long', concat([good, te.encode('fff5'), new Uint8Array(65_520)])],
['cut in a payload', concat([good, te.encode('000aab')])],
['cut in a length', concat([good, te.encode('00')])],
];
for (const [name, bytes] of cases) {
for (const [cut, chunks] of chunkings(bytes).slice(0, 3)) {
const want = pullPkt(chunks);
assert.notEqual(want.end, 'end', name);
const { stream, log } = recorded(chunks);
const { e, seen } = await failure(pktFrames(stream));
ply(e, 'protocol');
assert.equal(e.message, want.end.message, `${name} ${cut}`);
assert.deepEqual(seen, want.out, `${name} ${cut}`);
if (!log.closed) assert.deepEqual(log.cancelled, [e]);
}
}
});
test('inflateChunks: the output zlib inflates to, over every chunking; zlib and raw, capped and declared', async () => {
const data = concat([blob(70_000), te.encode('tail '.repeat(5000))]);
const variants = [
['zlib capped', zlib.deflateSync(data), {}],
['zlib stored', zlib.deflateSync(data, { level: 0 }), {}],
['zlib declared', zlib.deflateSync(data, { level: 9 }), { bound: declaredBound(data.length) }],
['raw', zlib.deflateRawSync(data), { wrap: rawWrap(), bound: cappedBound() }],
['empty', zlib.deflateSync(new Uint8Array(0)), {}],
];
for (const [name, member, opts] of variants) {
const want = name === 'empty' ? new Uint8Array(0) : data;
for (const [cut, chunks] of chunkings(member)) {
if (cut === 'bytes' && member.length > 20_000) continue;
const out = await collect(inflateChunks(recorded(chunks).stream, opts));
assert.deepEqual(Buffer.from(concat(out)), Buffer.from(want), `${name} ${cut}`);
assert.ok(out.every(c => c.length <= 16_384), 'chunks of at most INFLATE_CHUNK');
assert.deepEqual(Buffer.from(concat(await collect(inflateChunks(iterable(chunks).source, opts)))), Buffer.from(want), `${name} ${cut}: iterable`);
}
}
// a member fed one byte at a time, the slow path
const small = zlib.deflateSync(te.encode('hello, '.repeat(300)));
assert.equal(new TextDecoder().decode(concat(await collect(inflateChunks(recorded(Array.from(small, b => Uint8Array.of(b))).stream)))), 'hello, '.repeat(300));
});
test('inflateChunks: corrupt, truncated, trailing, oversized and mis-declared members are typed refusals', async () => {
const data = blob(50_000);
const member = zlib.deflateSync(data);
const flip = at => { const b = Uint8Array.from(member); b[at] ^= 0xff; return b; };
const cases = [
['bad header', flip(0), 'pack_corrupt', {}],
['bad adler', flip(member.length - 1), 'pack_corrupt', {}],
['truncated', member.subarray(0, member.length - 5), 'pack_corrupt', {}],
['empty', new Uint8Array(0), 'pack_corrupt', {}],
['trailing byte, same chunk', concat([member, Uint8Array.of(0)]), 'pack_corrupt', {}],
['trailing member', concat([member, member]), 'pack_corrupt', {}],
['past maxOutput', member, 'limit_exceeded', { bound: cappedBound(10_000) }],
['declared shorter', member, 'pack_corrupt', { bound: declaredBound(1000) }],
['declared longer', member, 'pack_corrupt', { bound: declaredBound(60_000) }],
];
for (const [name, bytes, code, opts] of cases) {
for (const [cut, chunks] of chunkings(bytes).slice(0, 3)) {
if (cut === 'bytes' && bytes.length > 2000) continue;
const { stream, log } = recorded(chunks);
const { e } = await failure(inflateChunks(stream, opts));
ply(e, code);
if (!log.closed) assert.deepEqual(log.cancelled, [e], `${name} ${cut}`);
}
}
// a trailing byte in a later chunk than the member's end
const { e } = await failure(inflateChunks(recorded([member, Uint8Array.of(1)]).stream));
ply(e, 'pack_corrupt');
assert.match(e.message, /1 bytes after the end/);
// empty chunks after the end are not bytes after it
assert.equal(concat(await collect(inflateChunks(recorded([member, new Uint8Array(0), new Uint8Array(0)]).stream))).length, data.length);
// bad options: the stream errors on the first read
ply((await failure(inflateChunks(recorded([member]).stream, { bound: cappedBound(-1) }))).e, 'bad_request');
});
test('inflateChunks agrees with the pull API on the same member (same chunks, byte for byte)', async () => {
const member = zlib.deflateSync(blob(100_000));
const st = unwrap(inflateInit({ bound: cappedBound(), wrap: zlibWrap(true) }));
const want = [];
inflateFeed(st, member);
for (let p = unwrap(inflatePull(st)); p.kind !== 'end'; p = unwrap(p.kind === 'need' ? inflateEnd(st) : inflatePull(st))) if (p.kind === 'chunk') want.push(p.bytes);
assert.deepEqual(await collect(inflateChunks(recorded([member]).stream)), want);
});
// The package root is imported here, inside the test, not at the top of the file: a static import would load every module
// src/index.ts re-exports before any test runs, making each of their module-level mutants static, run against all 412
// mutation tests (errors-doc.ts alone: 79 mutants, a 26-80 min shard, #313). Inside the test they are covered by it.
test('the adapters are exported from the package root', async () => {
const index = await import('../src/index.js');
assert.equal(index.scanPack, scanPack);
assert.equal(index.pktFrames, pktFrames);
assert.equal(index.inflateChunks, inflateChunks);
});