Artifactspublic-v0

ply

main
ply4 git calls336 ms52 KiB2 cachedprotocol v2
984 lines54 KiBRaw
1// packSubset (#235): a new pack of chosen entries of an existing pack, read through a range reader, compressed bytes
2// copied unchanged. The oracle is stock git: `git index-pack --strict` accepts every built pack, and `git verify-pack -v`
3// lists exactly the selected objects (by id) and nothing else.
4// - the whole pack in order is byte-identical to the source (OFS and REF packs written by git)
5// - an order that differs from the source: OFS distances rewritten, headers growing and shrinking
6// - REF passthrough (a REF-only pack, as Cloudflare Artifacts sends)
7// - an OFS base outside the selection: refused, or written as a REF_DELTA (`outsideBase: 'ref'`) and fixed thin by git
8// - extras appended; exact length up front; one range read per run of consecutive entries
9// - a range that is short or long errors the stream; a consumer that cancels cancels the range
10// - a pushPack round trip to git http-backend, fsck --strict clean
11// - memory: a source far larger than what is held (child process, --expose-gc)
12import { test, beforeAll as before, afterAll as after } from 'vitest';
13import assert from 'node:assert/strict';
14import { spawn } from 'node:child_process';
15import { MEMORY_CELL_FLAGS } from './memcell.js';
16import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
17import { join } from 'node:path';
18import { randomFillSync, createHash } from 'node:crypto';
19import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, fsckProblems, initRepo } from './server.js';
20import { packSubset } from '../src/packsubset.js';
21import { rangeFromBytes } from '../src/rangesource.js';
22import * as rangesource from '../src/rangesource.js';
23import { resolvePackRange } from '../src/packrangeresolve.js';
24import { loadTransferPack } from '../src/transferload.js';
25import { packIdentity } from '../src/packsaved.js';
26import { scanPackIndex } from '../src/packscanindex.js';
27import { hashObject, TYPE } from '../src/objects.js';
28import { te } from '../src/bytes.js';
29import { createTransport } from '../src/http.js';
30import { pushPackChecked, ZERO_OID } from '../src/push.js';
31import { unwrap } from './unwrap.js';
33const REF = 'refs/heads/main';
34let root, cleanup, srv, work, n = 0;
35const packs = {};
36const id = { name: 'a', email: 'a@x' };
37const commitAll = msg => { git(work, ['add', '.']); git(work, ['-c', `user.name=${id.name}`, '-c', `user.email=${id.email}`, 'commit', '-q', '-m', msg]); };
38before(async () => {
39 ({ root, cleanup } = tmpRoot());
40 srv = await startGitServer(root);
41 work = join(root, 'source');
42 mkdirSync(work);
43 initRepo(work);
44 // versions of one text file make deltas; a 300 KB random blob sits between them, so moving an entry across it moves
45 // its OFS distance across a varint byte boundary
46 const lines = Array.from({ length: 3000 }, (_, i) => `line ${i} of a file that changes a little each commit\n`).join('');
47 for (let v = 0; v < 6; v++) {
48 writeFileSync(join(work, 'text.txt'), `${lines}version ${v}\n`);
49 writeFileSync(join(work, `notes-${v % 2}.md`), `${lines.slice(0, 20_000)}note ${v}\n`);
50 if (v === 2) { const big = new Uint8Array(300_000); randomFillSync(big); writeFileSync(join(work, 'random.bin'), big); }
51 commitAll(`v${v}`);
52 }
53 const head = git(work, ['rev-parse', 'HEAD']).trim();
54 packs.head = head;
55 packs.ofs = new Uint8Array(gitBuf(work, ['pack-objects', '--revs', '--stdout', '-q', '--delta-base-offset'], `${head}\n`));
56 packs.ref = new Uint8Array(gitBuf(work, ['pack-objects', '--revs', '--stdout', '-q'], `${head}\n`));
57});
58after(async () => { await srv.close(); cleanup(); });
60const scan = async bytes => unwrap(await scanPackIndex(rangeStream(bytes)));
61const rangeStream = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
62const drain = async stream => {
63 const parts = [];
64 let len = 0;
65 const reader = stream.getReader();
66 for (;;) { const r = await reader.read(); if (r.done) break; parts.push(r.value); len += r.value.length; }
67 const out = new Uint8Array(len);
68 let at = 0;
69 for (const p of parts) { out.set(p, at); at += p.length; }
70 return out;
71};
72// `git verify-pack -v` of a pack: each object's id, type, offset and (for a delta) its base id
73function verifyPack(bytes) {
74 const dir = join(root, `v${n++}`);
75 mkdirSync(dir);
76 const file = join(dir, 'p.pack');
77 writeFileSync(file, bytes);
78 git(dir, ['index-pack', '-o', join(dir, 'p.idx'), file]);
79 const out = git(dir, ['verify-pack', '-v', join(dir, 'p.idx')]);
80 const rows = [];
81 for (const line of out.split('\n')) {
82 const m = /^([0-9a-f]{40}) (\w+)\s+(\d+) (\d+) (\d+)(?: (\d+) ([0-9a-f]{40}))?$/.exec(line);
83 if (m) rows.push({ oid: m[1], type: m[2], offset: Number(m[5]), base: m[7] ?? null });
84 }
85 return rows;
87// `git index-pack --strict` in a bare repo (optionally one that already has the source's objects, for a thin pack)
88function strictAccept(bytes, { thin = false } = {}) {
89 const bare = makeBareRepo(root, `s${n++}`);
90 if (thin) gitBuf(bare, ['index-pack', '--stdin'], Buffer.from(packs.ofs));
91 return gitBuf(bare, ['index-pack', '--strict', ...(thin ? ['--fix-thin'] : []), '--stdin'], Buffer.from(bytes)).toString();
93// The source's entry index for each object id, from git's own listing of the source.
94function indexByOid(srcBytes, idx) {
95 const at = new Map([...idx.entries()].map(e => [e.offset, e.index]));
96 return new Map(verifyPack(srcBytes).map(r => [r.oid, at.get(r.offset)]));
98// A range reader that counts reads and can be told to misbehave.
99function countingSource(bytes, { short = 0, long = 0 } = {}) {
100 const inner = rangeFromBytes(bytes);
101 const state = { reads: 0, cancelled: 0 };
102 return {
103 state,
104 size: bytes.length,
105 async read(offset, length) {
106 state.reads++;
107 const r = await inner.read(offset, length);
108 if (!r.ok) return r;
109 if (short === 0 && long === 0) return { ok: true, value: tap(r.value) };
110 const want = length - short + long;
111 const body = new Uint8Array(want);
112 body.set(bytes.subarray(offset, offset + Math.min(want, bytes.length - offset)));
113 return { ok: true, value: new ReadableStream({ start(c) { c.enqueue(body); c.close(); } }) };
114 },
115 };
116 function tap(stream) {
117 const reader = stream.getReader();
118 return new ReadableStream({
119 async pull(c) { const r = await reader.read(); if (r.done) c.close(); else c.enqueue(r.value); },
120 cancel() { state.cancelled++; return reader.cancel(); },
121 }, { highWaterMark: 0 });
122 }
124async function build(srcBytes, idx, selection, opts = {}) {
125 const src = countingSource(srcBytes);
126 const built = unwrap(await packSubset(src, idx, selection, opts));
127 const bytes = await drain(built.stream);
128 assert.equal(bytes.length, built.length, 'length is exact');
129 assert.equal(built.count, selection.length + (opts.extras?.length ?? 0));
130 return { bytes, built, reads: src.state.reads };
133test('the whole pack in order is the source, byte for byte (OFS and REF packs), in one range read', async () => {
134 for (const kind of ['ofs', 'ref']) {
135 const idx = await scan(packs[kind]);
136 const all = [...Array(idx.count).keys()];
137 const { bytes, reads } = await build(packs[kind], idx, all);
138 assert.deepEqual(bytes, packs[kind], kind);
139 assert.equal(reads, 1, `${kind}: one run, one read`);
140 strictAccept(bytes);
141 }
142});
144test('an order that differs from the source: OFS distances rewritten, headers grow and shrink, git accepts it', async () => {
145 const idx = await scan(packs.ofs);
146 const c = idx.columns;
147 // wholes first in reverse order, then every delta in an order with each base before it (a topological order)
148 const wholes = [...idx.entries()].filter(e => e.base.tag === 'none').map(e => e.index).reverse();
149 const placed = new Set(wholes);
150 const order = [...wholes];
151 let left = [...idx.entries()].filter(e => e.base.tag !== 'none').map(e => e.index).reverse();
152 while (left.length) {
153 const ready = left.filter(i => placed.has(c.baseIndex[i]));
154 assert.ok(ready.length, 'a base is always placeable');
155 for (const i of ready) { order.push(i); placed.add(i); }
156 left = left.filter(i => !placed.has(i));
157 }
158 const { bytes, reads } = await build(packs.ofs, idx, order);
159 const out = await scan(bytes);
160 // the same objects, by id, as the source
161 const src = verifyPack(packs.ofs).map(r => r.oid).sort();
162 assert.deepEqual(verifyPack(bytes).map(r => r.oid).sort(), src);
163 strictAccept(bytes);
164 // headers changed size: compare each entry's header length with its source entry's
165 let grew = 0;
166 for (const [k, i] of order.entries()) {
167 if (out.columns.headerBytes[k] > c.headerBytes[i]) grew++;
168 assert.equal(out.columns.compressed[k], c.compressed[i], 'compressed bytes copied unchanged');
169 }
170 assert.ok(grew > 0, 'some OFS distance grew a varint byte (shrinking is the next test)');
171 assert.ok(reads > 1, 'a reordered selection is several runs');
172});
174test('an OFS distance shrinks when the entries between base and delta are left out', async () => {
175 const idx = await scan(packs.ofs);
176 const c = idx.columns;
177 // the OFS delta farthest from its whole base (across the 300 KB random blob): select only the two of them
178 const far = [...idx.entries()].filter(e => e.base.tag === 'ofs' && c.storedType[c.baseIndex[e.index]] !== TYPE.OFS_DELTA)
179 .sort((a, b) => (b.offset - b.base.offset) - (a.offset - a.base.offset))[0];
180 assert.ok(far && far.offset - far.base.offset > 1 << 14, 'the fixture has a far delta');
181 const chain = [c.baseIndex[far.index], far.index];
182 const { bytes } = await build(packs.ofs, idx, chain);
183 const out = await scan(bytes);
184 const k = chain.indexOf(far.index);
185 assert.ok(out.columns.headerBytes[k] < c.headerBytes[far.index], 'the far delta header shrank');
186 assert.equal(verifyPack(bytes).length, chain.length);
187 strictAccept(bytes);
188});
190test('REF passthrough: a REF-only pack, a subset of whole objects and their REF deltas', async () => {
191 const idx = await scan(packs.ref);
192 assert.ok([...idx.entries()].some(e => e.base.tag === 'ref'), 'the fixture has REF deltas');
193 // every other run of entries, plus the base of every selected REF delta (so the pack is whole and git can list it)
194 const byOid = indexByOid(packs.ref, idx);
195 const chosen = new Set([...Array(idx.count).keys()].filter(i => i % 3 !== 1));
196 for (let grew = true; grew;) {
197 grew = false;
198 for (const i of [...chosen]) {
199 const e = idx.entry(i).value;
200 if (e.base.tag === 'ref' && !chosen.has(byOid.get(e.base.oid))) { chosen.add(byOid.get(e.base.oid)); grew = true; }
201 }
202 }
203 const sel = [...chosen].sort((a, b) => a - b);
204 assert.ok(sel.length < idx.count && sel.some(i => idx.entry(i).value.base.tag === 'ref'));
205 const { bytes } = await build(packs.ref, idx, sel);
206 const want = new Set([...byOid].filter(([, j]) => chosen.has(j)).map(([oid]) => oid));
207 assert.deepEqual(new Set(verifyPack(bytes).map(r => r.oid)), want);
208 const out = await scan(bytes);
209 assert.ok([...out.entries()].some(e => e.base.tag === 'ref'), 'REF deltas passed through as REF');
210 // a subset links to objects it leaves out: --strict in a repository that has them
211 strictAccept(bytes, { thin: true });
212});
214test('an OFS base outside the selection: refused by default; with outsideBase ref, a REF_DELTA git fixes thin', async () => {
215 const idx = await scan(packs.ofs);
216 const c = idx.columns;
217 const d = [...idx.entries()].find(e => e.base.tag === 'ofs' && c.storedType[c.baseIndex[e.index]] !== TYPE.OFS_DELTA);
218 assert.ok(d, 'the fixture has a delta on a whole object');
219 const refused = await packSubset(rangeFromBytes(packs.ofs), idx, [d.index]);
220 assert.equal(refused.ok, false);
221 assert.equal(refused.error.code, 'bad_request');
222 assert.match(refused.error.message, /base.*is not selected/);
223 // base written after the delta: also refused
224 const later = await packSubset(rangeFromBytes(packs.ofs), idx, [d.index, c.baseIndex[d.index]]);
225 assert.equal(later.ok, false);
226 assert.match(later.error.message, /written after it/);
227 const { bytes } = await build(packs.ofs, idx, [d.index], { outsideBase: 'ref' });
228 const out = await scan(bytes);
229 assert.equal(out.columns.storedType[0], TYPE.REF_DELTA);
230 assert.equal(out.entry(0).value.base.oid, idx.entry(c.baseIndex[d.index]).value.oid);
231 strictAccept(bytes, { thin: true });
232 // a base that is itself a delta has no id in the index: refused even with outsideBase ref
233 const dd = [...idx.entries()].find(e => e.base.tag === 'ofs' && c.storedType[c.baseIndex[e.index]] === TYPE.OFS_DELTA);
234 if (dd) {
235 const r = await packSubset(rangeFromBytes(packs.ofs), idx, [dd.index], { outsideBase: 'ref' });
236 assert.equal(r.ok, false);
237 assert.match(r.error.message, /id is unknown/);
238 }
239});
241test('extras are appended, deflated up front, and counted in the length', async () => {
242 const idx = await scan(packs.ofs);
243 const blob = te.encode('an extra object ply encoded itself\n');
244 const extra = { type: TYPE.BLOB, data: blob };
245 // whole blobs only: they link to nothing, so --strict has nothing missing
246 const wholes = [...idx.entries()].filter(e => e.storedType === TYPE.BLOB).map(e => e.index);
247 const { bytes } = await build(packs.ofs, idx, wholes, { extras: [extra] });
248 const oids = new Set(verifyPack(bytes).map(r => r.oid));
249 assert.ok(oids.has(hashObject('blob', blob)));
250 assert.equal(oids.size, wholes.length + 1);
251 strictAccept(bytes);
252 const bad = await packSubset(rangeFromBytes(packs.ofs), idx, wholes, { extras: [{ ...extra, base: { tag: 'ofs', index: 0 } }] });
253 assert.equal(bad.ok, false);
254 assert.equal(bad.error.code, 'bad_request');
255});
257test('the selection is checked before a byte is read: out of range, not an integer, twice', async () => {
258 const idx = await scan(packs.ofs);
259 for (const sel of [[idx.count], [-1], [0.5], [0, 0]]) {
260 const src = countingSource(packs.ofs);
261 const r = await packSubset(src, idx, sel);
262 assert.equal(r.ok, false, JSON.stringify(sel));
263 assert.equal(r.error.code, 'bad_request');
264 assert.equal(src.state.reads, 0);
265 }
266 const small = await packSubset(rangeFromBytes(packs.ofs.subarray(0, 100)), idx, [0]);
267 assert.equal(small.ok, false, 'a source smaller than the indexed pack');
268});
270test('refusals of the request or the index: more objects than a pack counts, a REF_DELTA the index gives no base id', async () => {
271 const idx = await scan(packs.ref);
272 // a pack's count is 32 bits: 2^32 - 1 selected (a sparse array, refused before it is walked) plus one extra
273 const huge = await packSubset(rangeFromBytes(packs.ref), idx, new Array(0xff_ff_ff_ff), { extras: [{ type: TYPE.BLOB, data: te.encode('x') }] });
274 assert.equal(huge.ok, false);
275 assert.equal(huge.error.code, 'bad_request');
276 assert.match(huge.error.message, /more than a pack can count/);
277 // an index (hand-built or damaged) whose entry() gives a REF_DELTA no base id
278 const r = [...idx.entries()].find(e => e.base.tag === 'ref');
279 const damaged = Object.create(idx);
280 damaged.entry = i => (i === r.index ? { tag: 'none' } : idx.entry(i));
281 const noBase = await packSubset(rangeFromBytes(packs.ref), damaged, [r.index]);
282 assert.equal(noBase.ok, false);
283 assert.equal(noBase.error.code, 'bad_request');
284 assert.match(noBase.error.message, /REF_DELTA with no base id/);
285});
287test('an index changed after planning: the stream errors instead of sending a length it did not plan', async () => {
288 const idx = await scan(packs.ofs);
289 const wholes = [...idx.entries()].filter(e => e.base.tag === 'none').map(e => e.index);
290 const built = unwrap(await packSubset(rangeFromBytes(packs.ofs), idx, wholes));
291 idx.columns.compressed[wholes.at(-1)] -= 1;
292 await assert.rejects(drain(built.stream), e => e.code === 'internal' && /wrote \d+ bytes, planned \d+/.test(e.message));
293});
295test('the selection is copied at the call: changing the array afterwards changes nothing git can see', async () => {
296 const idx = await scan(packs.ofs);
297 const c = idx.columns;
298 const entries = [...idx.entries()];
299 // git's own ids for the source's entries
300 const oidOf = new Map([...indexByOid(packs.ofs, idx)].map(([oid, i]) => [i, oid]));
301 // D is an OFS_DELTA on the whole object A; X is another whole object
302 const d = entries.find(e => e.base.tag === 'ofs' && entries[c.baseIndex[e.index]].base.tag === 'none').index;
303 const a = c.baseIndex[d];
304 const x = entries.find(e => e.base.tag === 'none' && e.index !== a).index;
305 const want = [oidOf.get(a), oidOf.get(x), oidOf.get(d)];
306 const listed = bytes => verifyPack(bytes).toSorted((p, q) => p.offset - q.offset).map(r => r.oid);
307 // left alone
308 const plain = await drain(unwrap(await packSubset(rangeFromBytes(packs.ofs), idx, [a, x, d])).stream);
309 assert.deepEqual(listed(plain), want);
310 // the caller reuses its array before the stream is read (a batching loop's scratch array): A and X swapped, so D's
311 // planned distance would land on X
312 const sel = [a, x, d];
313 const built = unwrap(await packSubset(rangeFromBytes(packs.ofs), idx, sel));
314 sel[0] = x;
315 sel[1] = a;
316 const changed = await drain(built.stream);
317 // git index-pack accepts it (verifyPack runs it) and lists the objects the call named, in the order it named them
318 assert.deepEqual(listed(changed), want);
319 assert.equal(verifyPack(changed).find(r => r.oid === oidOf.get(d)).base, oidOf.get(a), 'D is still a delta on A');
320 assert.deepEqual(changed, plain);
321 // more entries than the pack has cannot all be distinct and in it: refused before anything is sized by the selection
322 const over = await packSubset(rangeFromBytes(packs.ofs), idx, new Array(idx.count + 1).fill(a));
323 assert.equal(over.ok, false);
324 assert.equal(over.error.code, 'bad_request');
325});
327test('an extra a pack cannot hold is refused before the source sees a read', async () => {
328 const idx = await scan(packs.ofs);
329 const data = te.encode('an extra\n');
330 // gitformat-pack: an undeltified entry's type is 1 (commit), 2 (tree), 3 (blob) or 4 (tag); 0 is invalid and 5
331 // reserved; 6 and 7 are deltas, which need a base; a REF_DELTA's base is a 20-byte object name, 40 hex digits.
332 // (The reviewer's run had git index-pack reject the pack built from each of these.)
333 const invalid = [
334 ['type 9', { type: 9, data }],
335 ['type 0', { type: 0, data }],
336 ['type 5', { type: 5, data }],
337 ['type 7 (REF_DELTA) with no base', { type: 7, data }],
338 ['type 6 (OFS_DELTA) with no base', { type: 6, data }],
339 ['a fractional type', { type: 1.5, data }],
340 ['a REF base that is not hex', { type: TYPE.BLOB, data, base: { tag: 'ref', oid: 'zz' } }],
341 ['a REF base of 39 hex digits', { type: TYPE.BLOB, data, base: { tag: 'ref', oid: 'a'.repeat(39) } }],
342 ['a base of no known tag', { type: TYPE.BLOB, data, base: { tag: 'offset', index: 0 } }],
343 ['data that is not bytes', { type: TYPE.BLOB, data: 'an extra\n' }],
344 // not an object at all: there is no entry to write, and packSubset returns a Result (it never throws or rejects)
345 ['an extra that is null', null],
346 ['an extra that is undefined', undefined],
347 ['an extra that is a number', 7],
348 ];
349 for (const [name, extra] of invalid) {
350 const src = countingSource(packs.ofs);
351 // a valid extra first: the refusal names the one at fault
352 const r = await packSubset(src, idx, [0], { extras: [{ type: TYPE.BLOB, data }, extra] });
353 assert.equal(r.ok, false, name);
354 assert.equal(r.error.code, 'bad_request', name);
355 assert.equal(r.error.what, 'pack subset extras', name);
356 assert.match(r.error.message, /extra 1 /, name);
357 assert.equal(src.state.reads, 0, `${name}: no read`);
358 }
359});
361test('extras that is not an array is refused (bad_request), never thrown', async () => {
362 const idx = await scan(packs.ofs);
363 // The expected outcome is the contract's: packSubset returns a Result, so a caller's wrong value is a coded refusal
364 // (bad_request, what: 'pack subset extras'), not a TypeError out of the promise, and nothing is read for it.
365 const invalid = [['null', null], ['a number', 7], ['a string', 'ab'], ['a plain object', {}], ['an array-like', { length: 1, 0: { type: TYPE.BLOB, data: te.encode('x') } }], ['bytes', new Uint8Array(2)]];
366 for (const [name, extras] of invalid) {
367 const src = countingSource(packs.ofs);
368 const settled = await packSubset(src, idx, [0], { extras }).then(r => ({ r }), thrown => ({ thrown }));
369 assert.equal(settled.thrown, undefined, `${name}: rejected with ${String(settled.thrown)}`);
370 assert.equal(settled.r.ok, false, name);
371 assert.equal(settled.r.error.code, 'bad_request', name);
372 assert.equal(settled.r.error.what, 'pack subset extras', name);
373 assert.equal(src.state.reads, 0, `${name}: no read`);
374 }
375 // left out, or passed as the missing value, is no extras: the pack is the selection alone
376 for (const opts of [{}, { extras: undefined }]) {
377 const built = unwrap(await packSubset(rangeFromBytes(packs.ofs), idx, [0], opts));
378 assert.equal(built.count, 1);
379 }
380});
382test('rangeFromBytes refuses a range outside its bytes', async () => {
383 const src = rangeFromBytes(new Uint8Array(8));
384 for (const [o, l] of [[4, 5], [-1, 2], [9, 0], [0.5, 1]]) {
385 const r = await src.read(o, l);
386 assert.equal(r.ok, false, `${o}+${l}`);
387 assert.equal(r.error.code, 'bad_request');
388 assert.match(r.error.message, /outside 8 bytes/);
389 }
390 assert.ok((await src.read(0, 8)).ok);
391});
393test('runs of consecutive entries are one range read each', async () => {
394 const idx = await scan(packs.ofs);
395 const wholes = [...idx.entries()].filter(e => e.base.tag === 'none').map(e => e.index);
396 // runs in the selection (consecutive source indices written consecutively)
397 let runs = 0;
398 for (let k = 0; k < wholes.length; k++) if (k === 0 || wholes[k] !== wholes[k - 1] + 1) runs++;
399 const { reads } = await build(packs.ofs, idx, wholes);
400 assert.equal(reads, runs);
401});
403test('a range that serves too few or too many bytes errors the stream before the trailer', async () => {
404 const idx = await scan(packs.ofs);
405 const all = [...Array(idx.count).keys()];
406 for (const misbehave of [{ short: 1 }, { long: 1 }]) {
407 const built = unwrap(await packSubset(countingSource(packs.ofs, misbehave), idx, all));
408 await assert.rejects(drain(built.stream), e => e.code === 'pack_corrupt', JSON.stringify(misbehave));
409 }
410 const failing = { size: packs.ofs.length, read: async () => ({ ok: false, error: Object.assign(new Error('gone'), { code: 'network' }) }) };
411 const built = unwrap(await packSubset(failing, idx, all));
412 await assert.rejects(drain(built.stream), e => e.code === 'network');
413});
415test('a consumer that cancels the stream cancels the range read', async () => {
416 const idx = await scan(packs.ofs);
417 const src = countingSource(packs.ofs);
418 const built = unwrap(await packSubset(src, idx, [...Array(idx.count).keys()]));
419 const reader = built.stream.getReader();
420 await reader.read();
421 await reader.read();
422 await reader.cancel();
423 assert.equal(src.state.cancelled, 1);
424});
426// A hang is proved by a step counter, not a clock: `turns` event-loop turns pass while the promise is still pending.
427// (A cancel that works settles within a few microtasks of the call, before the first turn.)
428const settledWithin = async (promise, turns = 50) => {
429 let state = 'pending';
430 promise.then(() => { state = 'settled'; }, () => { state = 'settled'; });
431 for (let i = 0; i < turns && state === 'pending'; i++) await new Promise(r => setImmediate(r));
432 return state;
433};
434const readAll = async reader => { for (;;) { const r = await reader.read(); if (r.done) return; } };
435// A source that stops answering: `open: 'never'` leaves read() itself pending for ever; `open: 'stall'` opens a range
436// that serves `serve` bytes and then never another. `parked` resolves once the source is in that state.
437function stallingSource(bytes, { open, serve = 0 }) {
438 const state = { reads: 0, cancelled: 0, aborted: 0 };
439 let reached;
440 const parked = new Promise(r => { reached = r; });
441 return {
442 state, parked, size: bytes.length,
443 read(offset, length, signal) {
444 state.reads++;
445 signal?.addEventListener('abort', () => { state.aborted++; });
446 if (open === 'never') { reached(); return new Promise(() => {}); }
447 let at = offset;
448 return Promise.resolve({ ok: true, value: new ReadableStream({
449 pull(c) {
450 if (at - offset >= serve) { reached(); return new Promise(() => {}); }
451 const len = Math.min(serve - (at - offset), offset + length - at);
452 c.enqueue(bytes.subarray(at, at + len));
453 at += len;
454 return undefined;
455 },
456 cancel() { state.cancelled++; },
457 }, { highWaterMark: 0 }) });
458 },
459 };
462test('cancel settles while a range read is in flight, whether or not the source ever answers', async () => {
463 const idx = await scan(packs.ofs);
464 const all = [...Array(idx.count).keys()];
465 // the range is open and has stalled: cancel settles, and the range is cancelled
466 const stalled = stallingSource(packs.ofs, { open: 'stall', serve: 1000 });
467 const a = unwrap(await packSubset(stalled, idx, all)).stream.getReader();
468 const drainA = readAll(a);
469 await stalled.parked;
470 assert.equal(await settledWithin(a.cancel('stop')), 'settled', 'cancel() with a stalled range');
471 assert.equal(await settledWithin(drainA), 'settled', 'the read that was pending');
472 assert.equal(stalled.state.cancelled, 1, 'the stalled range is cancelled');
473 assert.equal(stalled.state.aborted, 1, 'the source is told through its signal');
474 // read() itself never resolves: cancel settles all the same, and the source is told through its signal
475 const never = stallingSource(packs.ofs, { open: 'never' });
476 const b = unwrap(await packSubset(never, idx, all)).stream.getReader();
477 const drainB = readAll(b);
478 await never.parked;
479 assert.equal(await settledWithin(b.cancel('stop')), 'settled', 'cancel() with read() pending');
480 assert.equal(await settledWithin(drainB), 'settled', 'the read that was pending');
481 assert.equal(never.state.reads, 1);
482 assert.equal(never.state.aborted, 1, 'a source that takes a signal is aborted');
483});
485test('the reason a cancelled range receives: the consumer\'s own, an AbortError when it gave none; the signal carries the same', async () => {
486 const idx = await scan(packs.ofs);
487 // DOM: AbortController.abort() with no reason sets an "AbortError" DOMException; a given reason is kept as it is
488 const cases = [['stop', r => r === 'stop'], [undefined, r => r instanceof DOMException && r.name === 'AbortError']];
489 for (const [given, expected] of cases) {
490 const seen = { cancels: [], aborts: [] };
491 let reached;
492 const parked = new Promise(r => { reached = r; });
493 const source = {
494 size: packs.ofs.length,
495 read(offset, length, signal) {
496 // a read given no signal records no abort, so the assertions below fail rather than this source throwing
497 signal?.addEventListener('abort', () => { seen.aborts.push(signal.reason); });
498 return Promise.resolve({ ok: true, value: new ReadableStream({ pull() { reached(); return new Promise(() => {}); }, cancel(why) { seen.cancels.push(why); } }, { highWaterMark: 0 }) });
499 },
500 };
501 const reader = unwrap(await packSubset(source, idx, [...Array(idx.count).keys()])).stream.getReader();
502 const drained = readAll(reader);
503 // no wait here is bare: a cancel that waits for the range (which never answers) fails the test, it does not hang it
504 assert.equal(await settledWithin(parked), 'settled', 'the range is open and has stalled');
505 assert.equal(await settledWithin(reader.cancel(given)), 'settled', `cancel(${String(given)}) with a stalled range`);
506 assert.equal(await settledWithin(drained), 'settled', 'the read that was pending');
507 assert.equal(seen.cancels.length, 1, String(given));
508 assert.ok(expected(seen.cancels[0]), `the range's cancel reason for ${String(given)}: ${String(seen.cancels[0])}`);
509 assert.deepEqual(seen.aborts, seen.cancels, 'read\'s signal aborts with the same reason');
510 }
511 // a range that is refused (it serves more bytes than were asked for) is cancelled with null, not with an error
512 const refused = [];
513 const long = { size: packs.ofs.length, read: (offset, length) => Promise.resolve({ ok: true, value: new ReadableStream({ pull(c) { c.enqueue(new Uint8Array(length + 1)); }, cancel(why) { refused.push(why); } }, { highWaterMark: 0 }) }) };
514 await assert.rejects(readAll(unwrap(await packSubset(long, idx, [...Array(idx.count).keys()])).stream.getReader()));
515 assert.deepEqual(refused, [null]);
516});
518test('a range opened after the stream was cancelled is cancelled, not left open', async () => {
519 const idx = await scan(packs.ofs);
520 // read() answers only when the test lets it, which is after the cancel
521 let answer, reached, cancelled = 0;
522 const parked = new Promise(r => { reached = r; });
523 const late = { size: packs.ofs.length, read: () => new Promise(r => { answer = r; reached(); }) };
524 const s = unwrap(await packSubset(late, idx, [...Array(idx.count).keys()])).stream.getReader();
525 const drained = readAll(s);
526 await parked;
527 assert.equal(await settledWithin(s.cancel()), 'settled');
528 assert.equal(await settledWithin(drained), 'settled');
529 answer({ ok: true, value: new ReadableStream({ cancel() { cancelled++; } }) });
530 await new Promise(r => setImmediate(r));
531 assert.equal(cancelled, 1, 'the late range');
532});
534test('pushPack over a source that stalls mid-range: an abort returns an outcome, and the ref did not move', async () => {
535 const idx = await scan(packs.ofs);
536 const src = stallingSource(packs.ofs, { open: 'stall', serve: 100_000 });
537 const built = unwrap(await packSubset(src, idx, [...Array(idx.count).keys()]));
538 const name = `stall${n++}`;
539 const bare = makeBareRepo(root, name);
540 const ac = new AbortController();
541 const t = unwrap(createTransport(`${srv.url}/${name}.git`, { signal: ac.signal }));
542 const pushed = pushPackChecked(t, { ref: REF, oldOid: ZERO_OID, newOid: packs.head, pack: built.stream, length: built.length, sideband: false });
543 await src.parked;
544 ac.abort(new Error('caller gave up'));
545 const { result: r } = await pushed;
546 // the request was under way with part of the pack sent: what the server did with it is not known to the client
547 assert.equal(r.ok, false);
548 assert.equal(r.error.code, 'ambiguous', r.error.message);
549 assert.equal(src.state.cancelled, 1, 'the stalled range is cancelled');
550 // git is the witness that nothing landed: a pack cut short cannot move a ref
551 assert.throws(() => git(bare, ['rev-parse', '--verify', '-q', REF]));
552}, 20_000);
554test('pushPack round trip: a reordered subset pushed to git http-backend lands, fsck --strict clean', async () => {
555 const idx = await scan(packs.ofs);
556 const c = idx.columns;
557 // children-first by type is what a transfer does; here: wholes (blobs, trees, commits) reversed, then deltas topologically
558 const wholes = [...idx.entries()].filter(e => e.base.tag === 'none').map(e => e.index).reverse();
559 const placed = new Set(wholes);
560 const order = [...wholes];
561 let left = [...idx.entries()].filter(e => e.base.tag !== 'none').map(e => e.index);
562 while (left.length) {
563 for (const i of left.filter(j => placed.has(c.baseIndex[j]))) { order.push(i); placed.add(i); }
564 left = left.filter(i => !placed.has(i));
565 }
566 const built = unwrap(await packSubset(rangeFromBytes(packs.ofs), idx, order));
567 const name = `push${n++}`;
568 const bare = makeBareRepo(root, name);
569 const t = unwrap(createTransport(`${srv.url}/${name}.git`));
570 const { result: r } = await pushPackChecked(t, { ref: REF, oldOid: ZERO_OID, newOid: packs.head, pack: built.stream, length: built.length, sideband: false });
571 assert.equal(r.ok, true, r.ok ? '' : r.error.message);
572 assert.equal(git(bare, ['rev-parse', REF]).trim(), packs.head);
573 assert.deepEqual(fsckProblems(bare, { args: ['--strict'] }), []);
574});
576test('memory: a subset of a source far larger than what is held', async () => {
577 // 48 entries of 4 MiB stored blobs (192 MiB of pack), every other one selected, drained slowly
578 const child = await new Promise((resolve, reject) => {
579 const p = spawn(process.execPath, [...MEMORY_CELL_FLAGS, '--import', './test/ts-resolve.mjs', 'test/pack-subset-child.mjs', String(4 << 20), '48'], { stdio: ['ignore', 'pipe', 'inherit'] });
580 let text = '';
581 p.stdout.on('data', d => { text += d; });
582 p.on('error', reject);
583 p.on('close', code => (code === 0 ? resolve(JSON.parse(text)) : reject(new Error(`child exited ${code}`))));
584 });
585 assert.equal(child.ok, true, child.error ?? '');
586 assert.equal(child.sent, child.length);
587 assert.ok(child.sourceBytes > 150 << 20, 'the source is far larger than the bound');
588 // held: one 64 KiB range chunk in flight plus small headers, not the pack (allowance for the drain's own chunk)
589 assert.ok(child.liveDelta < 1 << 20, `live ${child.liveDelta} B`);
590}, 120_000);
592// ---- every operation that reads the caller's storage, not only packSubset's stream: an abort settles it ----
593// The source is scripted, never slept on: read number `at` (0-based) stops answering, earlier reads are served from the
594// bytes. 'never' leaves read() itself pending for ever; 'stall' opens a range whose first chunk never comes. `parked`
595// resolves when the source is in that state, and the counters say what the operation told it afterwards.
596function stallAt(bytes, at, how) {
597 const real = rangeFromBytes(bytes);
598 const state = { reads: 0, cancelled: 0, aborted: 0 };
599 let reached;
600 const parked = new Promise(r => { reached = r; });
601 return {
602 state, parked, size: bytes.length,
603 read(offset, length, signal) {
604 const i = state.reads++;
605 if (i < at) return real.read(offset, length, signal);
606 // an operation that passes no signal still parks here: its test then fails at 'the abort settles it'
607 signal?.addEventListener('abort', () => { state.aborted++; });
608 if (how === 'never') { reached(); return new Promise(() => {}); }
609 return Promise.resolve({ ok: true, value: new ReadableStream({
610 pull() { reached(); return new Promise(() => {}); },
611 cancel() { state.cancelled++; },
612 }, { highWaterMark: 0 }) });
613 },
614 };
617// The expected outcome is the contract's, not ply's output: an operation its caller aborts ends with the error code
618// `aborted` (errors.ts: retry 'no'), and it ends when the signal fires, not when the storage answers (it never does
619// here). `stalledRead` is the read each operation is parked at, counted from the operation's own description:
620// packIdentity read 0 is the trailer
621// resolvePackRange read 0 is the first entry read back
622// loadTransferPack read 0 the trailer (packIdentity), read 1 the whole spool (opened here, read by the scan),
623// read 2 the first entry read back by the resolve
624const ABORTABLE = [
625 ['packIdentity', 0, (source, signal) => packIdentity(source, signal)],
626 ['resolvePackRange', 0, (source, signal, idx) => resolvePackRange(idx, source, { signal })],
627 ['loadTransferPack: the trailer', 0, (source, signal, idx, dst) => loadTransferPack(dst, source, { tag: 'none' }, { signal })],
628 ['loadTransferPack: the scan', 1, (source, signal, idx, dst) => loadTransferPack(dst, source, { tag: 'none' }, { signal })],
629 ['loadTransferPack: the resolve', 2, (source, signal, idx, dst) => loadTransferPack(dst, source, { tag: 'none' }, { signal })],
630];
631for (const [name, stalledRead, run] of ABORTABLE) {
632 for (const how of ['never', 'stall']) {
633 test(`${name}: an abort settles it 'aborted' while the storage never answers (${how})`, async () => {
634 const idx = await scan(packs.ofs);
635 const dst = unwrap(createTransport(`${srv.url}/none.git`));
636 const source = stallAt(packs.ofs, stalledRead, how);
637 const ac = new AbortController();
638 const op = run(source, ac.signal, idx, dst);
639 // bounded like every wait below: an operation that never reaches the read fails here, it does not hang
640 assert.equal(await settledWithin(source.parked), 'settled', 'the operation reached the read the case names');
641 assert.equal(source.state.reads, stalledRead + 1, 'parked at the read the case names');
642 assert.equal(await settledWithin(op, 20), 'pending', 'it is waiting for the storage');
643 ac.abort(new Error('stop'));
644 assert.equal(await settledWithin(op), 'settled', 'the abort settles it');
645 const r = await op;
646 assert.equal(r.ok, false);
647 assert.equal(r.error.code, 'aborted');
648 assert.equal(source.state.aborted, 1, 'the source is told through the signal of the read in flight');
649 if (how === 'stall') assert.equal(source.state.cancelled, 1, 'the range in flight is cancelled');
650 assert.equal(source.state.reads, stalledRead + 1, 'and nothing more is read');
651 });
652 }
655// A signal that aborted before the call: the operation is `aborted` and the storage is not asked for anything. The
656// expected values are the contract's, not ply's output: the error code `aborted` (errors.ts), and 0 on the test's own
657// counters (no read issued, so no range opened and none to cancel). One test for each entry point that takes a signal
658// and a source.
659const abortedBefore = () => { const ac = new AbortController(); ac.abort(new Error('before')); return ac.signal; };
660test('packIdentity: a signal aborted before the call is aborted before any storage read', async () => {
661 const source = countingSource(packs.ofs);
662 const op = packIdentity(source, abortedBefore());
663 assert.equal(await settledWithin(op), 'settled');
664 const r = await op;
665 assert.deepEqual({ code: r.ok ? 'ok' : r.error.code, ...source.state }, { code: 'aborted', reads: 0, cancelled: 0 });
666});
667test('resolvePackRange: a signal aborted before the call is aborted before any storage read, a pack of whole objects included', async () => {
668 // git's own pack of the same history with the delta search off: every object is stored whole, so resolving it needs
669 // no read at all and only a check at the entry can see the abort. verify-pack says which pack has deltas.
670 const whole = new Uint8Array(gitBuf(work, ['pack-objects', '--revs', '--stdout', '-q', '--window=0'], `${packs.head}\n`));
671 assert.equal(verifyPack(whole).filter(r => r.base !== null).length, 0, 'the whole-object pack has no delta');
672 assert.ok(verifyPack(packs.ofs).some(r => r.base !== null), 'the other pack has deltas');
673 const seen = {};
674 for (const [name, bytes] of [['deltas', packs.ofs], ['whole objects', whole]]) {
675 const idx = await scan(bytes);
676 const source = countingSource(bytes);
677 const op = resolvePackRange(idx, source, { signal: abortedBefore() });
678 assert.equal(await settledWithin(op), 'settled', name);
679 const r = await op;
680 seen[name] = { code: r.ok ? 'ok' : r.error.code, ...source.state };
681 }
682 assert.deepEqual(seen, {
683 deltas: { code: 'aborted', reads: 0, cancelled: 0 },
684 'whole objects': { code: 'aborted', reads: 0, cancelled: 0 },
685 });
686});
688test('scanPackIndex: an abort settles it while its stream, or its iterator, never yields; the source is asked to stop', async () => {
689 // a ReadableStream that serves 100 bytes and then nothing
690 const src = stallingSource(packs.ofs, { open: 'stall', serve: 100 });
691 const stream = (await src.read(0, packs.ofs.length, new AbortController().signal)).value;
692 const a = new AbortController();
693 const scanA = scanPackIndex(stream, { signal: a.signal });
694 await src.parked;
695 assert.equal(await settledWithin(scanA, 20), 'pending');
696 a.abort(new Error('stop'));
697 assert.equal(await settledWithin(scanA), 'settled', 'a stalled stream');
698 const ra = await scanA;
699 assert.equal(ra.ok, false);
700 assert.equal(ra.error.code, 'aborted');
701 assert.equal(src.state.cancelled, 1, 'the stream is cancelled');
702 // an async iterable whose next() never resolves
703 const seen = { next: 0, returned: 0 };
704 let reached;
705 const parked = new Promise(r => { reached = r; });
706 const iterable = { [Symbol.asyncIterator]: () => ({
707 next() { seen.next++; reached(); return new Promise(() => {}); },
708 return() { seen.returned++; return new Promise(() => {}); },
709 }) };
710 const b = new AbortController();
711 const scanB = scanPackIndex(iterable, { signal: b.signal });
712 await parked;
713 b.abort(new Error('stop'));
714 assert.equal(await settledWithin(scanB), 'settled', 'a stalled iterator whose return() never settles either');
715 const rb = await scanB;
716 assert.equal(rb.ok, false);
717 assert.equal(rb.error.code, 'aborted');
718 assert.deepEqual(seen, { next: 1, returned: 1 });
719 // a signal that aborted before the call: nothing is waited for at all
720 const c = new AbortController();
721 c.abort(new Error('before'));
722 const scanC = scanPackIndex({ [Symbol.asyncIterator]: () => ({ next: () => new Promise(() => {}) }) }, { signal: c.signal });
723 assert.equal(await settledWithin(scanC), 'settled', 'a signal aborted before the call');
724 const rc = await scanC;
725 assert.equal(rc.ok, false);
726 assert.equal(rc.error.code, 'aborted');
727});
729test('what the storage answers after the abort is dropped: a late range is cancelled, a late rejection is not unhandled', async () => {
730 const idx = await scan(packs.ofs);
731 const unhandled = [];
732 const note = e => { unhandled.push(e); };
733 process.on('unhandledRejection', note);
734 try {
735 for (const late of ['opens', 'rejects']) {
736 const state = { reads: 0, cancelled: 0 };
737 let answer, refuse, reached;
738 const parked = new Promise(r => { reached = r; });
739 const source = { size: packs.ofs.length, read() {
740 state.reads++;
741 reached();
742 return new Promise((resolve, reject) => { answer = resolve; refuse = reject; });
743 } };
744 const ac = new AbortController();
745 const op = resolvePackRange(idx, source, { signal: ac.signal });
746 await parked;
747 ac.abort(new Error('stop'));
748 assert.equal(await settledWithin(op), 'settled', late);
749 assert.equal((await op).error.code, 'aborted', late);
750 // only now does the storage answer
751 if (late === 'opens') answer({ ok: true, value: new ReadableStream({ cancel() { state.cancelled++; } }) });
752 else refuse(new Error('the storage failed, late'));
753 for (let i = 0; i < 20; i++) await new Promise(r => setImmediate(r));
754 assert.equal(state.cancelled, late === 'opens' ? 1 : 0, `${late}: a range that opened late is cancelled, not left open`);
755 assert.equal(state.reads, 1, late);
756 }
757 assert.deepEqual(unhandled, []);
758 } finally {
759 process.off('unhandledRejection', note);
760 }
761});
763test('haltOn: every await in flight is released at the abort, and a finished operation leaves nothing on the signal', async () => {
764 const ac = new AbortController();
765 // count the listeners on the caller's signal with the platform's own add/remove
766 const listeners = new Set();
767 const add = ac.signal.addEventListener.bind(ac.signal);
768 const remove = ac.signal.removeEventListener.bind(ac.signal);
769 ac.signal.addEventListener = (type, fn, o) => { listeners.add(fn); add(type, fn, o); };
770 ac.signal.removeEventListener = (type, fn, o) => { listeners.delete(fn); remove(type, fn, o); };
771 const done = rangesource.haltOn(ac.signal);
772 assert.equal(listeners.size, 1);
773 // not halted: the race is the promise's own outcome, a value or a rejection
774 assert.equal(await done.race(Promise.resolve(7), () => {}), 7);
775 await assert.rejects(done.race(Promise.reject(new Error('mine')), () => {}), /mine/);
776 done.release();
777 assert.equal(listeners.size, 0, 'release() takes the listener off a signal that outlives the operation');
778 // two awaits in flight at once: both are released, and each late value goes to its own `late`
779 const halt = rangesource.haltOn(ac.signal);
780 const late = [];
781 let first, second;
782 const one = halt.race(new Promise(r => { first = r; }), v => { late.push(v); });
783 const two = halt.race(new Promise(r => { second = r; }), v => { late.push(v); });
784 ac.abort(new Error('stop'));
785 assert.deepEqual(await Promise.all([one, two]), [rangesource.HALTED, rangesource.HALTED]);
786 first('a');
787 second('b');
788 for (let i = 0; i < 5; i++) await new Promise(r => setImmediate(r));
789 assert.deepEqual(late, ['a', 'b']);
790 // after the abort nothing is waited for, not even a promise that never settles
791 assert.equal(await halt.race(new Promise(() => {}), () => {}), rangesource.HALTED);
792});
794test('a source whose read answers with its Result itself, not a promise of it, is read as a promise of it would be', async () => {
795 // A JavaScript caller's source: read() returns { ok, value } synchronously. Every await on it was a bare await before
796 // the abort work, and a bare await takes a plain value. Expected values, none from the code under test: git's own
797 // pack bytes; the pack format's trailer (the SHA-1 of every byte before it, here node:crypto's) and the pack's
798 // length; and the object ids `git verify-pack -v` lists for the pack.
799 const idx = await scan(packs.ofs);
800 const all = [...Array(idx.count).keys()];
801 const source = { size: packs.ofs.length, read: (offset, length) => ({ ok: true, value: rangeStream(packs.ofs.subarray(offset, offset + length)) }) };
802 // bounded like every wait in this file: a drain or a call that never ends fails here, it does not hang
803 const built = unwrap(await packSubset(source, idx, all));
804 const drained = drain(built.stream);
805 assert.equal(await settledWithin(drained, 2000), 'settled', 'the drain ends');
806 assert.deepEqual(await drained, packs.ofs);
807 const trailer = createHash('sha1').update(packs.ofs.subarray(0, packs.ofs.length - 20)).digest('hex');
808 const byOid = indexByOid(packs.ofs, idx);
809 assert.equal(byOid.size, idx.count, 'git lists every entry of the pack');
810 for (const signal of [undefined, new AbortController().signal]) {
811 const identity = packIdentity(source, signal);
812 assert.equal(await settledWithin(identity, 2000), 'settled', 'packIdentity ends');
813 assert.deepEqual(unwrap(await identity), { trailer, bytes: packs.ofs.length });
814 const resolving = resolvePackRange(idx, source, signal === undefined ? {} : { signal });
815 assert.equal(await settledWithin(resolving, 2000), 'settled', 'resolvePackRange ends');
816 const resolved = unwrap(await resolving);
817 for (const [oid, i] of byOid) assert.equal(resolved.oid(i).value, oid);
818 // loadTransferPack opens the whole spool through the same read (no index on the destination: it scans)
819 const dst = unwrap(createTransport(`${srv.url}/none.git`));
820 const loading = loadTransferPack(dst, source, { tag: 'none' }, signal === undefined ? {} : { signal });
821 assert.equal(await settledWithin(loading, 2000), 'settled', 'loadTransferPack ends');
822 const loaded = unwrap(await loading);
823 for (const [oid, i] of byOid) assert.equal(loaded.from.resolved.oid(i).value, oid);
824 }
825});
827test('a range that is a ReadableStream subclass serving its bytes through its own getReader() is read through that reader', async () => {
828 // The subclass has no underlying source: its bytes come only from the reader its own getReader() returns (the idiom
829 // test/push-pack.test.js uses for a caller-controlled reader). Expected values, none from the code under test: git's
830 // own pack bytes; one read, because the whole pack in order is one run; one getReader call, one per range.
831 const idx = await scan(packs.ofs);
832 const all = [...Array(idx.count).keys()];
833 const seen = { reads: 0, readers: 0 };
834 const source = { size: packs.ofs.length, read(offset, length) {
835 seen.reads++;
836 const inner = rangeStream(packs.ofs.subarray(offset, offset + length)).getReader();
837 class Served extends ReadableStream { getReader() { seen.readers++; return inner; } }
838 return Promise.resolve({ ok: true, value: new Served() });
839 } };
840 const built = unwrap(await packSubset(source, idx, all));
841 const drained = drain(built.stream);
842 // bounded: a range whose bytes are never read fails here, it does not hang
843 assert.equal(await settledWithin(drained, 2000), 'settled', 'the drain ends');
844 assert.deepEqual(await drained, packs.ofs);
845 assert.deepEqual(seen, { reads: 1, readers: 1 });
846});
848// ---- a range reader that breaks the streams contract when asked to cancel ----
849// The two ways a hand-written reader does: cancel() answers with no promise, or throws where it stands. The range is
850// the caller's, and its failure to cancel is its own: ply asks it to stop and waits for nothing, so neither may reach
851// anyone as an error or hold an operation past its abort.
852const BROKEN_CANCEL = [
853 ['returns undefined', () => undefined],
854 ['throws where it stands', () => { throw new Error('sync cancel'); }],
855];
856// What node reports thrown out of every caller's reach while `run` runs. An error thrown inside an AbortSignal listener
857// (where an abort asks the range in flight to cancel) is caught by no one: node raises it as an uncaughtException on a
858// later tick, which ends a process that has no handler for it. The turns after `run` let that tick come.
859const uncaughtDuring = async run => {
860 const seen = [];
861 const onUncaught = e => { seen.push(String(e)); };
862 process.on('uncaughtException', onUncaught);
863 try {
864 await run();
865 for (let i = 0; i < 5; i++) await new Promise(r => setImmediate(r));
866 } finally {
867 process.off('uncaughtException', onUncaught);
868 }
869 return seen;
870};
871for (const [name, broken] of BROKEN_CANCEL) {
872 test(`cancelRange asks a range to stop and does not throw when its cancel() ${name}`, () => {
873 // Expected values are the test's own: the one reason it gave, seen once by the range's cancel()
874 const reason = new Error('stop');
875 const reasons = [];
876 assert.doesNotThrow(() => rangesource.cancelRange({ cancel(why) { reasons.push(why); return broken(why); } }, reason));
877 assert.deepEqual(reasons, [reason]);
878 });
880 test(`packSubset over a range read through its own reader, whose cancel() ${name}: the consumer's cancel settles and nothing is thrown`, async () => {
881 // The range is a ReadableStream subclass that defines getReader, so it is read through the reader that returns.
882 // Expected values, none from the code under test: the streams contract for the consumer (cancel() fulfils with
883 // undefined), node's own uncaughtException event (none raised), and the test's counter (the range was asked to stop).
884 const idx = await scan(packs.ofs);
885 const seen = { cancels: 0 };
886 const source = { size: packs.ofs.length, read(offset, length) {
887 let at = offset;
888 const reader = {
889 // 256 bytes a read, so the consumer's cancel lands with the range open and most of it unread
890 read() {
891 const n = Math.min(256, offset + length - at);
892 if (n === 0) return Promise.resolve({ done: true, value: undefined });
893 at += n;
894 return Promise.resolve({ done: false, value: packs.ofs.subarray(at - n, at) });
895 },
896 cancel(why) { seen.cancels++; return broken(why); },
897 releaseLock() {},
898 };
899 class Served extends ReadableStream { getReader() { return reader; } }
900 return Promise.resolve({ ok: true, value: new Served() });
901 } };
902 let cancelled = 'not reached';
903 const uncaught = await uncaughtDuring(async () => {
904 const consumer = unwrap(await packSubset(source, idx, [...Array(idx.count).keys()])).stream.getReader();
905 for (let got = 0; got < 2000;) {
906 const chunk = consumer.read();
907 // bounded like every wait in this file: a read or a cancel that never ends fails here, it does not hang
908 assert.equal(await settledWithin(chunk), 'settled', 'a chunk of the new pack');
909 const r = await chunk;
910 assert.equal(r.done, false, 'the pack is longer than what is read before the cancel');
911 got += r.value.length;
912 }
913 const cancelling = consumer.cancel(new Error('consumer left'));
914 assert.equal(await settledWithin(cancelling), 'settled', 'cancel()');
915 cancelled = await cancelling.then(v => ({ fulfilled: v }), e => ({ rejected: String(e) }));
916 });
917 assert.deepEqual({ cancelled, uncaught }, { cancelled: { fulfilled: undefined }, uncaught: [] });
918 assert.ok(seen.cancels >= 1, 'the open range was asked to stop');
919 });
921 // The expected outcome is the contract's, as for ABORTABLE above: `aborted` when the signal fires, though the read in
922 // flight never answers. The counters are the test's own: one read issued, and its range asked once to stop.
923 const SIGNALLED = [
924 ['packIdentity', (source, signal) => packIdentity(source, signal)],
925 ['resolvePackRange', (source, signal, idx) => resolvePackRange(idx, source, { signal })],
926 ];
927 for (const [op, run] of SIGNALLED) {
928 test(`${op}: an abort settles it 'aborted' with a read in flight on a range whose cancel() ${name}`, async () => {
929 const idx = await scan(packs.ofs);
930 const seen = { reads: 0, cancels: 0 };
931 let reached;
932 const parked = new Promise(r => { reached = r; });
933 const source = { size: packs.ofs.length, read() {
934 seen.reads++;
935 const reader = {
936 read() { reached(); return new Promise(() => {}); },
937 cancel(why) { seen.cancels++; return broken(why); },
938 releaseLock() {},
939 };
940 class Stalled extends ReadableStream { getReader() { return reader; } }
941 return Promise.resolve({ ok: true, value: new Stalled() });
942 } };
943 const ac = new AbortController();
944 let outcome = 'not reached';
945 const uncaught = await uncaughtDuring(async () => {
946 const running = run(source, ac.signal, idx);
947 assert.equal(await settledWithin(parked), 'settled', 'the operation is reading the range');
948 assert.equal(await settledWithin(running, 20), 'pending', 'it is waiting for the storage');
949 ac.abort(new Error('stop'));
950 assert.equal(await settledWithin(running), 'settled', 'the abort settles it');
951 const r = await running;
952 outcome = r.ok ? 'ok' : r.error.code;
953 });
954 assert.deepEqual({ outcome, uncaught, ...seen }, { outcome: 'aborted', uncaught: [], reads: 1, cancels: 1 });
955 });
956 }
959test('a chunk read is not a promise race: a whole pack served 256 bytes at a time is drained without one Promise.race', async () => {
960 // Expected values, neither from the code under test: 0 is the owner's decision (no race per read), and the bytes are
961 // git's own pack. The count is the platform's Promise.race, wrapped for the length of the drain.
962 const idx = await scan(packs.ofs);
963 const all = [...Array(idx.count).keys()];
964 let chunks = 0;
965 const source = { size: packs.ofs.length, async read(offset, length) {
966 let at = offset;
967 return { ok: true, value: new ReadableStream({ pull(c) {
968 const n = Math.min(256, offset + length - at);
969 if (n === 0) { c.close(); return; }
970 chunks++;
971 c.enqueue(packs.ofs.subarray(at, at + n));
972 at += n;
973 } }, { highWaterMark: 0 }) };
974 } };
975 const built = unwrap(await packSubset(source, idx, all));
976 const platformRace = Promise.race;
977 let races = 0;
978 Promise.race = function race(list) { races++; return platformRace.call(this, list); };
979 let bytes;
980 try { bytes = await drain(built.stream); } finally { Promise.race = platformRace; }
981 assert.equal(chunks, Math.ceil(packs.ofs.length / 256), 'every chunk of the pack was read, 256 bytes at a time');
982 assert.equal(races, 0);
983 assert.deepEqual(bytes, packs.ofs);
984});
Diagnostics4 git calls · 336 ms
CallStatusSizeTime0336 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit15 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=15
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=15
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=38434
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=38434
git.pkt.payload.max=38339
git.sideband.band1.bytes=38338
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=63
git.pack.bytes=38338
git.pack.ofs_delta=0
git.pack.ref_delta=8
git.pack.chain.max=8
git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77
git.pack.full=55
Open pack map
fetch20014 KiB319 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=319
git.response.bytes=14786
git.response.cache=MISS
git.response.duration_ms=319
git.fetch.wants=0bf929a6239ef7a8027a9f4798128145c4a7ae5b
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=14786
git.pkt.payload.max=14765
git.sideband.band1.bytes=14764
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e19ffc59c03cd0
Request
cdd5ee4d…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format