GitHubivarvong

ply

main
ply6 git calls234 ms36 KiB3 cachedprotocol v2
690 lines48 KiBRaw
1// Saved pack index (#260): encodeSavedIndex / decodeSavedIndex and loadTransferPack.
2// - round trip, REF-only and OFS packs, with and without resolved columns, oids on and off: every column, every
3// entry view, find() and the resolver's stats come back equal; a transfer planned from the decoded index is the
4// one planned from a fresh scan
5// - size: exactly 64 + 31/entry (+20 oids, +20 REF bases, +37 resolved, +72 stats) + 20 bytes
6// - refusals, each its own tag: not a saved index, another version, every truncation length, every single-byte
7// corruption (the check), another pack (trailer, length), and each malformed field with a valid check
8// - encode refusals: an index short of its header count, resolved columns of another length, no trailer
9// - loadTransferPack: none → scanned; saved → used, reading only the spool's last 20 bytes; no resolved columns →
10// resolved again; stale / corrupt / truncated / foreign saved bytes → scanned, with the refusal on the result and
11// the span; a spool too small for a pack, a spool whose read fails, a spool that is not a pack
12// - a resumed transfer whose every request loads the saved index reaches the head a fresh one does, at every step
13// boundary, against git http-backend with fsck on
14import { test, beforeAll as before, afterAll as after } from 'vitest';
15import assert from 'node:assert/strict';
16import { mkdirSync, writeFileSync } from 'node:fs';
17import { join } from 'node:path';
18import { randomFillSync, createHash } from 'node:crypto';
19import { deflateSync } from 'node:zlib';
20import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, fsckProblems, initRepo } from './server.js';
21import { scanPackIndex, scannedPackOf } from '../src/packscanindex.js';
22import { resolvePackRange } from '../src/packrangeresolve.js';
23import { rangeFromBytes } from '../src/rangesource.js';
24import { createTransport } from '../src/http.js';
25import { planTransfer } from '../src/transferplan.js';
26import { runTransfer } from '../src/transfer.js';
27import { encodeSavedIndex, decodeSavedIndex, describeRefusal, packIdentity, SAVED_INDEX_VERSION, SAVED_INDEX_ENTRY_BYTES, SAVED_INDEX_FIXED_BYTES } from '../src/packsaved.js';
28import { loadTransferPack } from '../src/transferload.js';
29import { Sha1 } from '../src/sha1.js';
30import { Tracer, arrayExporter, flattenLine } from '../src/otel.js';
31import { some, none } from '../src/result.js';
32import { unwrap } from './unwrap.js';
33import { settledWithin, stallAt, abortedBefore, codeOf } from './abortcells.js';
35const must = r => { if (r.ok) return r.value; throw Object.assign(new Error(r.error.message ?? JSON.stringify(r.error)), { code: r.error.code }); };
36const streamOf = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
37const ident = ['-c', 'user.name=Ann Author', '-c', 'user.email=ann@x'];
38const REF = 'refs/heads/main';
39const BUDGET = { maxRequestBytes: 200_000, maxObjects: 300, maxObjectBytes: 1 << 20, maxCompressedObjectBytes: 1 << 20, reserveBytes: 0, reserveObjects: 0 };
41let root, cleanup, srv, n = 0, work, src;
42before(async () => {
43 ({ root, cleanup } = tmpRoot());
44 srv = await startGitServer(root);
45 work = join(root, 'source');
46 mkdirSync(join(work, 'a', 'b'), { recursive: true });
47 initRepo(work);
48 for (let i = 0; i < 8; i++) {
49 const blob = new Uint8Array(40_000 + i * 700);
50 randomFillSync(blob);
51 writeFileSync(join(work, i % 2 ? 'a' : join('a', 'b'), `r${i}.bin`), blob);
52 }
53 const base = new Uint8Array(30_000);
54 randomFillSync(base);
55 for (let v = 0; v < 5; v++) { base[v * 997] ^= 0xff; writeFileSync(join(work, 'a', `chain${v}.bin`), base); }
56 writeFileSync(join(work, 'small.txt'), 'hello\n');
57 git(work, ['add', '.']);
58 git(work, [...ident, 'commit', '-q', '-m', 'saved index']);
59 const head = git(work, ['rev-parse', 'HEAD']).trim();
60 const objs = git(work, ['rev-list', '--objects', '--no-walk', head]);
61 src = {
62 head,
63 commit: new Uint8Array(gitBuf(work, ['cat-file', 'commit', head])),
64 lsTree: git(work, ['ls-tree', '-r', head]),
65 packs: {
66 ref: new Uint8Array(gitBuf(work, ['pack-objects', '--stdout', '-q', '--depth=50'], objs)),
67 ofs: new Uint8Array(gitBuf(work, ['pack-objects', '--stdout', '-q', '--depth=50', '--delta-base-offset'], objs)),
68 },
69 };
70});
71after(async () => { await srv?.close(); cleanup?.(); });
73const fresh = async (bytes, scan = {}) => {
74 const pack = must(await scanPackIndex(streamOf(bytes), scan));
75 const source = rangeFromBytes(bytes);
76 return { source, pack, resolved: must(await resolvePackRange(pack, source)) };
77};
78const idOf = from => ({ trailer: from.pack.trailer, bytes: from.pack.bytes });
79const COLS = ['offset', 'headerBytes', 'compressed', 'size', 'storedType', 'type', 'baseIndex', 'oids', 'refs'];
80const RCOLS = ['oid', 'type', 'base', 'depth', 'size'];
81function assertSame(a, b) {
82 for (const k of COLS) assert.deepEqual([...b.pack.columns[k]], [...a.pack.columns[k]], k);
83 for (const k of ['version', 'count', 'trailer', 'bytes']) assert.equal(b.pack[k], a.pack[k], k);
84 assert.deepEqual([...b.pack.entries()], [...a.pack.entries()]);
85 assert.deepEqual(b.pack.entry(-1), none);
86 assert.deepEqual(b.pack.entry(a.pack.count), none);
88function assertSameResolved(a, b) {
89 for (const k of RCOLS) assert.deepEqual([...b.columns[k]], [...a.columns[k]], k);
90 assert.deepEqual(b.stats, a.stats);
91 assert.equal(b.count, a.count);
92 assert.equal(b.heldBytes, a.heldBytes);
93 for (let i = -1; i <= a.count; i++) { assert.deepEqual(b.oid(i), a.oid(i)); assert.deepEqual(b.type(i), a.type(i)); }
94 for (let i = 0; i < a.count; i++) { const id = a.oid(i).value; assert.deepEqual(b.find(id), a.find(id)); }
95 assert.deepEqual(b.find('0'.repeat(40)), none);
96 assert.deepEqual(b.find('not an id'), none);
98// Rewrite the check after an edit, so only the structural checks can refuse it.
99const rechecked = bytes => { const out = Uint8Array.from(bytes); out.set(new Sha1().update(out.subarray(0, out.length - 20)).digest(), out.length - 20); return out; };
100const sizeOf = (n, { oids = true, refs = false, resolved = true } = {}) => SAVED_INDEX_FIXED_BYTES - (resolved ? 0 : 72) + n * (SAVED_INDEX_ENTRY_BYTES.index + (oids ? 20 : 0) + (refs ? 20 : 0) + (resolved ? SAVED_INDEX_ENTRY_BYTES.resolved : 0));
102test('round trip: REF-only and OFS packs, with and without resolved columns, oids on and off; the stated size', async () => {
103 for (const kind of ['ref', 'ofs']) {
104 const from = await fresh(src.packs[kind]);
105 const refs = kind === 'ref';
106 const bytes = must(encodeSavedIndex(from.pack, some(from.resolved)));
107 assert.equal(bytes.length, sizeOf(from.pack.count, { refs }), `${kind}: ${bytes.length} B for ${from.pack.count} entries`);
108 const back = must(decodeSavedIndex(bytes, idOf(from)));
109 assertSame(from, back);
110 assertSameResolved(from.resolved, back.resolved.value);
111 // a plan from the decoded index is the plan from the fresh one
112 assert.deepEqual(must(planTransfer(back.pack, back.resolved.value, { budget: BUDGET, ref: REF, commit: src.commit })), must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit })));
113 // without resolved columns
114 const bare = must(encodeSavedIndex(from.pack, none));
115 assert.equal(bare.length, sizeOf(from.pack.count, { refs, resolved: false }));
116 const b2 = must(decodeSavedIndex(bare, idOf(from)));
117 assertSame(from, b2);
118 assert.deepEqual(b2.resolved, none);
119 // encoding is deterministic, and a decoded index encodes to the same bytes
120 assert.deepEqual(must(encodeSavedIndex(back.pack, back.resolved)), bytes);
121 }
122 // oids off (an OFS pack resolves its types without them): no oid column
123 const off = await fresh(src.packs.ofs, { oids: false });
124 const bytes = must(encodeSavedIndex(off.pack, none));
125 assert.equal(bytes.length, sizeOf(off.pack.count, { oids: false, resolved: false }));
126 const back = must(decodeSavedIndex(bytes, idOf(off)));
127 assertSame(off, back);
128 assert.ok([...back.pack.entries()].every(e => e.oid === null));
129});
131test('refusals: not a saved index, another version, every truncation, every corrupted byte, another pack', async () => {
132 const from = await fresh(src.packs.ref);
133 const bytes = must(encodeSavedIndex(from.pack, some(from.resolved)));
134 const want = idOf(from);
135 const tags = {};
136 const count = r => { assert.equal(r.ok, false); tags[r.error.tag] = (tags[r.error.tag] ?? 0) + 1; assert.ok(describeRefusal(r.error).length > 0); return r.error; };
137 assert.equal(count(decodeSavedIndex(new TextEncoder().encode('PACK and more bytes than a header needs, sixty-four and twenty more bytes in all, so long'), want)).tag, 'not_saved_index');
138 assert.equal(count(decodeSavedIndex(new Uint8Array(3), want)).tag, 'not_saved_index');
139 const v2 = Uint8Array.from(bytes); v2[4] = SAVED_INDEX_VERSION + 1;
140 assert.deepEqual(count(decodeSavedIndex(rechecked(v2), want)), { tag: 'version', version: SAVED_INDEX_VERSION + 1 });
141 // every truncation: a prefix of the magic, of the header, of the body
142 for (let len = 0; len < bytes.length; len++) {
143 const e = count(decodeSavedIndex(bytes.subarray(0, len), want));
144 assert.ok(e.tag === 'truncated' || (len === 0 && e.tag === 'truncated'), `len ${len}: ${e.tag}`);
145 }
146 // every single byte flipped past the version: the check refuses it (or the header it changed says so first)
147 for (let i = 6; i < bytes.length; i++) {
148 const b = Uint8Array.from(bytes); b[i] ^= 0x01;
149 const e = count(decodeSavedIndex(b, want));
150 // a flip in the count's top bytes (16-19) claims more entries than maxPackObjects: refused by count first (#299)
151 assert.ok(['corrupt', 'truncated'].includes(e.tag) || (e.tag === 'limit_exceeded' && i >= 16 && i < 20), `byte ${i}: ${e.tag}`);
152 if (i >= 20) assert.equal(e.tag, 'corrupt', `byte ${i}`);
153 }
154 // another pack: the trailer, the length
155 const other = await fresh(src.packs.ofs);
156 assert.equal(count(decodeSavedIndex(bytes, idOf(other))).tag, 'other_pack');
157 assert.deepEqual(count(decodeSavedIndex(bytes, { ...want, bytes: want.bytes + 1 })), { tag: 'other_pack', saved: want, want: { ...want, bytes: want.bytes + 1 } });
158 assert.ok(tags.truncated >= bytes.length, `${tags.truncated} truncations`);
159 assert.equal(tags.corrupt + (tags.truncated - bytes.length) + (tags.limit_exceeded || 0), bytes.length - 6);
160});
162test('malformed with a valid check: each field a scan could not produce is refused by name', async () => {
163 const from = await fresh(src.packs.ref);
164 const ofs = await fresh(src.packs.ofs);
165 const enc = f => must(encodeSavedIndex(f.pack, some(f.resolved)));
166 const n = from.pack.count;
167 const dv = b => new DataView(b.buffer, b.byteOffset, b.byteLength);
168 // offsets into the body, as packsaved.ts lays it out
169 const at = (count, flags) => {
170 const offset = 64, compressed = offset + count * 8, size = compressed + count * 8, baseIndex = size + count * 8;
171 const headerBytes = baseIndex + count * 4, storedType = headerBytes + count, type = storedType + count, oids = type + count;
172 const refs = oids + count * 20, resolved = refs + (flags & 2 ? count * 20 : 0);
173 return { offset, compressed, size, baseIndex, headerBytes, storedType, type, oids, refs, resolved, rBase: resolved + count * 8, rDepth: resolved + count * 12, rType: resolved + count * 16 + count * 20, stats: resolved + count * 37 };
174 };
175 const refAt = at(n, 7);
176 const firstDelta = [...from.pack.entries()].find(e => e.storedType === 7).index;
177 const firstWhole = [...from.pack.entries()].find(e => e.storedType !== 7).index;
178 const cases = [
179 ['flags (unknown bit)', b => { dv(b).setUint16(6, dv(b).getUint16(6, true) | 0x80, true); }, 'flags', -1],
180 ['flags (refs off with REF entries)', null, 'length', -1],
181 ['reserved', b => { b[60] = 1; }, 'reserved', -1],
182 ['count', b => { dv(b).setUint32(12, n + 1, true); }, 'count', -1],
183 ['resolved.count', b => { dv(b).setUint32(48, n - 1, true); }, 'resolved.count', -1],
184 ['offset', b => { dv(b).setFloat64(refAt.offset + 8, 13, true); }, 'offset', 1],
185 ['compressed', b => { dv(b).setFloat64(refAt.compressed, 1.5, true); }, 'size', 0],
186 ['size', b => { dv(b).setFloat64(refAt.size, -1, true); }, 'size', 0],
187 ['headerBytes', b => { b[refAt.headerBytes] = 0; }, 'size', 0],
188 ['storedType', b => { b[refAt.storedType] = 5; }, 'storedType', 0],
189 ['type of a whole object', b => { b[refAt.type + firstWhole] = b[refAt.type + firstWhole] === 3 ? 2 : 3; }, 'type', firstWhole],
190 ['baseIndex of a whole object', b => { dv(b).setInt32(refAt.baseIndex + firstWhole * 4, 0, true); }, 'type', firstWhole],
191 ['baseIndex of a REF delta past the end', b => { dv(b).setInt32(refAt.baseIndex + firstDelta * 4, n, true); }, 'baseIndex', firstDelta],
192 // its baseIndex is left the scan's (review R2: one moved to a whole object with another id is refused as baseIndex
193 // before its type is read; the tampered table below has that case), so only the type disagrees with its base's
194 ['type of a REF delta', b => { b[refAt.type + firstDelta] = b[refAt.type + firstDelta] === 3 ? 2 : 3; }, 'type', firstDelta],
195 ['resolved.type', b => { b[refAt.rType] = 7; }, 'resolved.type', 0],
196 ['resolved.base self', b => { dv(b).setInt32(refAt.rBase + 4, 1, true); }, 'resolved.base', 1],
197 ['resolved.base below thin', b => { dv(b).setInt32(refAt.rBase, -3, true); }, 'resolved.base', 0],
198 ['resolved.depth', b => { dv(b).setInt32(refAt.rDepth + firstWhole * 4, 1, true); }, 'resolved.depth', firstWhole],
199 ['resolved.size', b => { dv(b).setFloat64(refAt.resolved, 0.5, true); }, 'resolved.size', 0],
200 ['resolved.stats', b => { dv(b).setFloat64(refAt.stats, Number.NaN, true); }, 'resolved.stats', -1],
201 ];
202 for (const [name, edit, field, index] of cases) {
203 if (edit === null) continue;
204 const b = enc(from);
205 edit(b);
206 const r = decodeSavedIndex(rechecked(b), idOf(from));
207 assert.equal(r.ok, false, name);
208 assert.deepEqual(r.error, { tag: 'malformed', field, index }, name);
209 }
210 // the REF flag cleared on a pack with REF entries: the buffer is longer than its header gives
211 const noRefs = enc(from);
212 dv(noRefs).setUint16(6, 5, true);
213 assert.deepEqual(decodeSavedIndex(rechecked(noRefs), idOf(from)).error, { tag: 'malformed', field: 'length', index: -1 });
214 // a longer buffer than its header gives, with a valid check
215 const long = new Uint8Array(enc(from).length + 8); long.set(enc(from).subarray(0, enc(from).length - 20));
216 assert.deepEqual(decodeSavedIndex(rechecked(long), idOf(from)).error, { tag: 'malformed', field: 'length', index: -1 });
217 // an OFS base that is not earlier, and the last entry not ending at the trailer
218 const ofsAt = at(ofs.pack.count, 5);
219 const delta = [...ofs.pack.entries()].find(e => e.storedType === 6).index;
220 const b = enc(ofs);
221 dv(b).setInt32(ofsAt.baseIndex + delta * 4, delta, true);
222 assert.deepEqual(decodeSavedIndex(rechecked(b), idOf(ofs)).error, { tag: 'malformed', field: 'baseIndex', index: delta });
223 const last = enc(ofs);
224 const k = ofs.pack.count - 1;
225 dv(last).setFloat64(ofsAt.compressed + k * 8, dv(last).getFloat64(ofsAt.compressed + k * 8, true) - 1, true);
226 assert.deepEqual(decodeSavedIndex(rechecked(last), idOf(ofs)).error, { tag: 'malformed', field: 'offset', index: ofs.pack.count });
227 // the REF flag set on a pack without REF entries (OFS), with room for the column
228 const fake = enc(ofs);
229 const grown = new Uint8Array(fake.length + ofs.pack.count * 20);
230 grown.set(fake.subarray(0, ofsAt.refs));
231 grown.set(fake.subarray(ofsAt.refs, fake.length - 20), ofsAt.refs + ofs.pack.count * 20);
232 dv(grown).setUint16(6, 7, true);
233 assert.deepEqual(decodeSavedIndex(rechecked(grown), idOf(ofs)).error, { tag: 'malformed', field: 'flags', index: -1 });
234});
236// Review finding saved-index-resolved-base-cycle: a saved index is stored state, so untrusted. Each tampered index below
237// has its trailing SHA-1 recomputed (the check is unkeyed), so only the structural checks can refuse it. The expected
238// refusal comes from the pack format, not from ply: a delta's object is its base's with one more delta applied (depth
239// = base depth + 1, same type), an OFS_DELTA names its base by position, a REF_DELTA by the base's id, a whole object
240// has no base, and the objects are git's own (pack-objects above).
241test('tampered resolved columns with a valid check: cycles, depth lies and bases the scan contradicts are malformed', async () => {
242 const from = await fresh(src.packs.ref);
243 const ofs = await fresh(src.packs.ofs);
244 const enc = f => must(encodeSavedIndex(f.pack, some(f.resolved)));
245 const dv = b => new DataView(b.buffer, b.byteOffset, b.byteLength);
246 // where the resolved columns start: 64 B header, 31 B per entry, 20 B of ids, 20 B of REF base ids in a REF pack
247 const rAt = (count, refs) => { const r = 64 + count * (31 + 20 + (refs ? 20 : 0)); return { size: r, base: r + count * 8, depth: r + count * 12, oid: r + count * 16, type: r + count * 36 }; };
248 const setBase = (b, at, i, v) => dv(b).setInt32(at.base + i * 4, v, true);
249 const setDepth = (b, at, i, v) => dv(b).setInt32(at.depth + i * 4, v, true);
250 const refusal = (f, edit, name) => {
251 const b = enc(f);
252 edit(b);
253 const r = decodeSavedIndex(rechecked(b), idOf(f));
254 assert.equal(r.ok, false, `${name}: accepted`);
255 assert.equal(r.error.tag, 'malformed', name);
256 return `${r.error.field}@${r.error.index}`;
257 };
258 const R = rAt(from.pack.count, true);
259 const rc = from.resolved.columns;
260 const all = [...Array(from.pack.count).keys()];
261 const deltas = all.filter(i => from.pack.columns.storedType[i] === 7);
262 const blobs = all.filter(i => from.pack.columns.storedType[i] === 3);
263 assert.ok(deltas.length >= 3 && blobs.length >= 3, 'the fixture has three REF deltas and three whole blobs');
264 const [d0, d1, d2] = deltas;
265 // the finding's bytes: two deltas resting on each other, depth 1 each
266 assert.match(refusal(from, b => { setBase(b, R, d0, d1); setBase(b, R, d1, d0); setDepth(b, R, d0, 1); setDepth(b, R, d1, 1); }, '2-cycle'), /^resolved\.(base|depth)@/u);
267 // a longer cycle whose depths agree on every link but the one that closes it
268 assert.match(refusal(from, b => { setBase(b, R, d0, d1); setBase(b, R, d1, d2); setBase(b, R, d2, d0); setDepth(b, R, d0, 3); setDepth(b, R, d1, 2); setDepth(b, R, d2, 1); }, '3-cycle'), /^resolved\.(base|depth)@/u);
269 assert.equal(refusal(from, b => { setBase(b, R, d0, d0); }, 'self base'), `resolved.base@${d0}`);
270 // depth lies with every base left honest: one more, one less, none, negative
271 for (const lie of [rc.depth[d0] + 1, rc.depth[d0] - 1, 0, -1]) assert.equal(refusal(from, b => { setDepth(b, R, d0, lie); }, `depth ${lie}`), `resolved.depth@${d0}`);
272 // a delta resolved to no base, and a whole object given one
273 assert.equal(refusal(from, b => { setBase(b, R, d0, -1); setDepth(b, R, d0, 0); }, 'delta without a base'), `resolved.base@${d0}`);
274 assert.equal(refusal(from, b => { setBase(b, R, blobs[0], blobs[1]); setDepth(b, R, blobs[0], 1); }, 'whole object with a base'), `resolved.base@${blobs[0]}`);
275 // a REF delta resting on a whole blob moved to another whole blob: depth 1 and type blob still agree, the base's id does not
276 const onWhole = deltas.find(i => from.pack.columns.storedType[rc.base[i]] === 3);
277 const otherBlob = blobs.find(i => i !== rc.base[onWhole]);
278 assert.equal(refusal(from, b => { setBase(b, R, onWhole, otherBlob); }, 'REF base with another id'), `resolved.base@${onWhole}`);
279 // the scan found that base in the pack, so it is not a thin base
280 assert.equal(from.pack.columns.baseIndex[onWhole], rc.base[onWhole]);
281 assert.equal(refusal(from, b => { setBase(b, R, onWhole, -2); setDepth(b, R, onWhole, 1); }, 'thin base the scan found in the pack'), `resolved.base@${onWhole}`);
282 // #260 review R2. The scan's own base column (after the 64 B header and three f64 columns, an i32 per entry): a REF
283 // delta's baseIndex is the whole object whose id the delta names. The fixture says so itself (both ids are in the
284 // index), and git wrote both: the REF id is in the pack's bytes, the whole object's id is its SHA-1.
285 const hex = (col, i) => Buffer.from(col.subarray(i * 20, i * 20 + 20)).toString('hex');
286 assert.equal(hex(from.pack.columns.oids, from.pack.columns.baseIndex[onWhole]), hex(from.pack.columns.refs, onWhole));
287 assert.notEqual(hex(from.pack.columns.oids, otherBlob), hex(from.pack.columns.refs, onWhole));
288 const scanBaseAt = i => 64 + from.pack.count * 24 + i * 4;
289 assert.equal(dv(enc(from)).getInt32(scanBaseAt(onWhole), true), from.pack.columns.baseIndex[onWhole], 'the offset is the scan baseIndex column');
290 // moved to a whole blob with another id: type (blob) and range still agree, and the resolved base is left honest
291 assert.equal(refusal(from, b => { dv(b).setInt32(scanBaseAt(onWhole), otherBlob, true); }, 'scan REF base with another id'), `baseIndex@${onWhole}`);
292 // the same lie told in both columns, and told with no resolved columns saved
293 assert.equal(refusal(from, b => { dv(b).setInt32(scanBaseAt(onWhole), otherBlob, true); setBase(b, R, onWhole, otherBlob); }, 'scan and resolved REF base with another id'), `baseIndex@${onWhole}`);
294 const bare = must(encodeSavedIndex(from.pack, none));
295 dv(bare).setInt32(scanBaseAt(onWhole), otherBlob, true);
296 assert.deepEqual(decodeSavedIndex(rechecked(bare), idOf(from)).error, { tag: 'malformed', field: 'baseIndex', index: onWhole });
297 // #260 re-review REQ 2 (owner decision 4). The same pack indexed without ids (scan oids: false; flags 2 refs, +4
298 // with resolved columns). A scan that keeps no ids finds no REF base: its base column is -1 for every REF delta
299 // (asserted on the scan's own output), so a recorded base is one no scan gives and has no id to be checked against.
300 // Both forgeries name a whole blob, with the type column made to agree (blob): the blob the ids-on index above shows
301 // is not the object the delta names, and the one that is. With and without resolved columns.
302 const noIds = await fresh(src.packs.ref, { oids: false });
303 assert.equal(noIds.pack.columns.oids.length, 0);
304 assert.ok(deltas.every(i => noIds.pack.columns.baseIndex[i] === -1), 'without ids the scan finds no REF base');
305 // the type column: after the base column (i32) and headerBytes, storedType (u8 each)
306 const scanTypeAt = i => 64 + from.pack.count * 30 + i;
307 for (const resolved of [some(noIds.resolved), none]) {
308 const honest = must(encodeSavedIndex(noIds.pack, resolved));
309 assert.equal(dv(honest).getUint16(6, true), resolved.tag === 'some' ? 6 : 2, 'flags: REF ids, no ids');
310 assert.equal(honest[scanTypeAt(onWhole)], 0, 'the offset is the scan type column: a REF delta with no base found has no type');
311 assert.ok(decodeSavedIndex(honest, idOf(noIds)).ok, 'the honest index without ids is used');
312 for (const to of [otherBlob, from.pack.columns.baseIndex[onWhole]]) {
313 const forged = Uint8Array.from(honest);
314 dv(forged).setInt32(scanBaseAt(onWhole), to, true);
315 forged[scanTypeAt(onWhole)] = 3;
316 const r = decodeSavedIndex(rechecked(forged), idOf(noIds));
317 assert.equal(r.ok, false, `REF scan base ${to} recorded without ids (${resolved.tag} resolved): accepted`);
318 assert.deepEqual(r.error, { tag: 'malformed', field: 'baseIndex', index: onWhole });
319 }
320 }
321 // types, ids and sizes the scan knows: a delta's type is its base's, a whole object's are the scan's
322 assert.equal(refusal(from, b => { b[R.type + onWhole] = 2; }, 'delta type'), `resolved.type@${onWhole}`);
323 assert.equal(refusal(from, b => { b[R.type + blobs[0]] = 2; }, 'whole type'), `resolved.type@${blobs[0]}`);
324 assert.equal(refusal(from, b => { b[R.oid + blobs[0] * 20] ^= 1; }, 'whole id'), `resolved.oid@${blobs[0]}`);
325 // ids are compared whole: one that differs in any single byte (each of the 20) is another id
326 for (let k = 0; k < 20; k++) assert.equal(refusal(from, b => { b[R.oid + blobs[0] * 20 + k] ^= 0x80; }, `whole id byte ${k}`), `resolved.oid@${blobs[0]}`);
327 assert.equal(refusal(from, b => { dv(b).setFloat64(R.size + blobs[0] * 8, rc.size[blobs[0]] + 1, true); }, 'whole size'), `resolved.size@${blobs[0]}`);
328 // OFS: the base is the scan's baseIndex and nothing else: a later entry, an earlier one, a cycle
329 const O = rAt(ofs.pack.count, false);
330 const oc = ofs.resolved.columns;
331 const ofsAll = [...Array(ofs.pack.count).keys()];
332 const od = ofsAll.filter(i => ofs.pack.columns.storedType[i] === 6);
333 assert.ok(od.length >= 2, 'the fixture has two OFS deltas');
334 const [o0, o1] = od;
335 const later = ofsAll.find(i => i > o0 && ofs.pack.columns.storedType[i] === 3 && i !== oc.base[o0]);
336 const earlier = ofsAll.find(i => i < o0 && ofs.pack.columns.storedType[i] === 3 && i !== oc.base[o0]);
337 assert.equal(ofs.pack.columns.baseIndex[o0], oc.base[o0]);
338 assert.ok(later !== undefined, 'a whole blob after the first OFS delta');
339 for (const to of [later, earlier]) {
340 if (to === undefined) continue;
341 assert.equal(refusal(ofs, b => { setBase(b, O, o0, to); setDepth(b, O, o0, 1); }, `OFS base ${to}`), `resolved.base@${o0}`);
342 }
343 assert.match(refusal(ofs, b => { setBase(b, O, o0, o1); setBase(b, O, o1, o0); setDepth(b, O, o0, 1); setDepth(b, O, o1, 1); }, 'OFS 2-cycle'), /^resolved\.base@/u);
344 // depth is held to the chain limit the caller resolves under: the honest index passes at its deepest chain, not below
345 const deepest = Math.max(...rc.depth);
346 assert.ok(deepest >= 1);
347 assert.ok(decodeSavedIndex(enc(from), idOf(from), deepest).ok);
348 assert.deepEqual(decodeSavedIndex(enc(from), idOf(from), deepest - 1).error, { tag: 'malformed', field: 'resolved.depth', index: all.find(i => rc.depth[i] === deepest) });
349 // the tampered index is rescanned by loadTransferPack, never used
350 const cyc = enc(from);
351 setBase(cyc, R, d0, d1); setBase(cyc, R, d1, d0); setDepth(cyc, R, d0, 1); setDepth(cyc, R, d1, 1);
352 const loaded = must(await loadTransferPack(traced().t, from.source, some(rechecked(cyc))));
353 assert.equal(loaded.loaded.tag, 'scanned');
354 assert.equal(loaded.loaded.refused.value.tag, 'malformed');
355 assertSameResolved(from.resolved, loaded.from.resolved);
356});
358test('encode refusals: an index short of its count, resolved columns of another length, no trailer', async () => {
359 const from = await fresh(src.packs.ref);
360 const short = scannedPackOf({ version: 2, count: from.pack.count + 1, trailer: from.pack.trailer, bytes: from.pack.bytes, heldBytes: 0 }, from.pack.columns);
361 assert.match(encodeSavedIndex(short, none).error.message, /holds \d+ of the pack's \d+ entries/);
362 assert.match(encodeSavedIndex(from.pack, some({ ...from.resolved, count: 1 })).error.message, /1 resolved entries/);
363 const noTrailer = scannedPackOf({ version: 2, count: from.pack.count, trailer: '', bytes: from.pack.bytes, heldBytes: 0 }, from.pack.columns);
364 assert.equal(encodeSavedIndex(noTrailer, none).error.code, 'bad_request');
365});
367// A spool that counts what is read through it.
368function counted(bytes) {
369 const inner = rangeFromBytes(bytes);
370 const c = { reads: 0, bytes: 0 };
371 return { c, source: { size: inner.size, read: async (o, l) => { c.reads++; c.bytes += l; return inner.read(o, l); } } };
373function traced() {
374 const lines = [];
375 const tracer = new Tracer({ exporter: arrayExporter(lines) });
376 const t = unwrap(createTransport(`${srv.url}/none.git`, { tracer }));
377 return { t, spans: () => lines.map(l => flattenLine(l)).flat() };
380test('loadTransferPack: scanned with none; saved used reading only the trailer; resolved again; stale and corrupt saved bytes rescanned, with why', async () => {
381 const bytes = src.packs.ref;
382 const { t, spans } = traced();
383 const first = counted(bytes);
384 const scanned = must(await loadTransferPack(t, first.source, none));
385 assert.deepEqual(scanned.loaded, { tag: 'scanned', refused: none });
386 assert.ok(first.c.bytes >= bytes.length, `a scan reads the spool: ${first.c.bytes} B`);
387 const again = counted(bytes);
388 const saved = must(await loadTransferPack(t, again.source, some(scanned.saved)));
389 assert.deepEqual(saved.loaded, { tag: 'saved' });
390 assert.deepEqual(again.c, { reads: 1, bytes: 20 }, 'only the trailer is read');
391 assert.equal(saved.saved, scanned.saved);
392 assertSame(scanned.from, saved.from);
393 assertSameResolved(scanned.from.resolved, saved.from.resolved);
394 // no resolved columns saved: the index is used, the resolve runs
395 const half = counted(bytes);
396 const resolved = must(await loadTransferPack(t, half.source, some(must(encodeSavedIndex(scanned.from.pack, none)))));
397 assert.deepEqual(resolved.loaded, { tag: 'resolved' });
398 assert.deepEqual(resolved.saved, scanned.saved);
399 // refused saved bytes: each is rescanned and says why
400 const other = must(await loadTransferPack(t, rangeFromBytes(src.packs.ofs), none));
401 const flipped = Uint8Array.from(scanned.saved); flipped[100] ^= 1;
402 const v9 = Uint8Array.from(scanned.saved); v9[4] = 9;
403 for (const [bad, tag] of [[other.saved, 'other_pack'], [flipped, 'corrupt'], [scanned.saved.subarray(0, 99), 'truncated'], [new Uint8Array(200), 'not_saved_index'], [rechecked(v9), 'version']]) {
404 const r = must(await loadTransferPack(t, rangeFromBytes(bytes), some(bad)));
405 assert.equal(r.loaded.tag, 'scanned');
406 assert.equal(r.loaded.refused.value.tag, tag);
407 assert.deepEqual(r.saved, scanned.saved, `${tag}: the fresh encoding is the one to keep`);
408 }
409 const got = spans().filter(s => s.name === 'git.transfer.load');
410 assert.equal(got.length, 9);
411 assert.deepEqual(got.map(s => s.attrs['git.transfer.load.from']), ['scanned', 'saved', 'resolved', 'scanned', 'scanned', 'scanned', 'scanned', 'scanned', 'scanned']);
412 assert.deepEqual(got.map(s => s.attrs['git.transfer.load.refused']), [undefined, undefined, undefined, undefined, 'other_pack', 'corrupt', 'truncated', 'not_saved_index', 'version']);
413 for (const s of got) { assert.equal(s.attrs['git.pack.trailer'], scanned.from.pack.trailer === s.attrs['git.pack.trailer'] ? scanned.from.pack.trailer : other.from.pack.trailer); assert.equal(typeof s.attrs['git.transfer.load.saved_bytes'], 'number'); }
414});
416// #260 review R1. A hand-built pack of one chain 100 deep: 101 blobs, entry 0 whole, entry i a REF_DELTA on entry i-1
417// (the delta copies the first 40 bytes of its base and inserts the rest). Stock git is the judge that it is an honest
418// pack and of how deep it is (index-pack --strict accepts it; verify-pack -v prints each delta's depth). 64 is
419// DEFAULT_LIMITS.maxDeltaChain, so only a caller's raised limit resolves it. (git pack-objects --depth=200 over 120
420// versions of one file chained 43 deep at --window=20 and 5 at --window=2: its heuristics do not promise a depth.)
421test('loadTransferPack: a pack chained deeper than 64 is used from its saved index under the caller\'s raised maxDeltaChain (scanned -> saved -> saved)', async () => {
422 const DEPTH = 100;
423 const sha1 = b => createHash('sha1').update(b).digest();
424 const entryHeader = (type, size) => { const out = [(type << 4) | (size & 15)]; let rest = size >> 4; while (rest > 0) { out[out.length - 1] |= 0x80; out.push(rest & 0x7f); rest >>= 7; } return Buffer.from(out); };
425 const versions = Array.from({ length: DEPTH + 1 }, (_, i) => Buffer.from(`c ${'x'.repeat(40)} v${i}\n`));
426 const ids = versions.map(v => sha1(Buffer.concat([Buffer.from(`blob ${v.length}\0`), v])));
427 // base size, result size (both under 128: one byte each), copy 40 bytes from offset 0, insert the rest
428 const deltaOf = (base, target) => Buffer.concat([Buffer.from([base.length, target.length, 0x90, 40, target.length - 40]), target.subarray(40)]);
429 const head = Buffer.alloc(12);
430 head.write('PACK'); head.writeUInt32BE(2, 4); head.writeUInt32BE(DEPTH + 1, 8);
431 const parts = [head, entryHeader(3, versions[0].length), deflateSync(versions[0])];
432 for (let i = 1; i <= DEPTH; i++) { const d = deltaOf(versions[i - 1], versions[i]); parts.push(entryHeader(7, d.length), ids[i - 1], deflateSync(d)); }
433 const body = Buffer.concat(parts);
434 const bytes = new Uint8Array(Buffer.concat([body, sha1(body)]));
435 const dir = join(root, `deep${n++}`);
436 mkdirSync(dir);
437 initRepo(dir);
438 writeFileSync(join(dir, 'deep.pack'), bytes);
439 git(dir, ['index-pack', '--strict', 'deep.pack']);
440 const depths = git(dir, ['verify-pack', '-v', 'deep.idx']).split('\n').map(l => /^[0-9a-f]{40} \w+\s+\d+ \d+ \d+ (\d+) [0-9a-f]{40}$/u.exec(l)).filter(m => m !== null).map(m => Number(m[1]));
441 const deepest = Math.max(...depths);
442 assert.equal(deepest, DEPTH, 'git reads the chain as 100 deep');
443 const { t } = traced();
444 const limits = { maxDeltaChain: 200 };
445 const source = rangeFromBytes(bytes);
446 // the default limit refuses the pack itself: the raised one is what the caller needs to read it at all
447 assert.equal((await loadTransferPack(t, source, none)).error.code, 'limit_exceeded');
448 const first = must(await loadTransferPack(t, source, none, { resolve: { limits } }));
449 assert.equal(first.loaded.tag, 'scanned');
450 assert.equal(Math.max(...first.from.resolved.columns.depth), deepest);
451 const second = must(await loadTransferPack(t, source, some(first.saved), { resolve: { limits } }));
452 assert.deepEqual(second.loaded, { tag: 'saved' });
453 const third = must(await loadTransferPack(t, source, some(second.saved), { resolve: { limits } }));
454 assert.deepEqual(third.loaded, { tag: 'saved' });
455 assert.deepEqual([...third.saved], [...first.saved]);
456 assertSameResolved(first.from.resolved, third.from.resolved);
457 // the limit is the caller's, both ways: at the pack's deepest chain it loads, one below it the saved index is refused
458 // and the rescan's resolve refuses the pack, as it does with no saved index
459 assert.deepEqual(must(await loadTransferPack(t, source, some(first.saved), { resolve: { limits: { maxDeltaChain: deepest } } })).loaded, { tag: 'saved' });
460 assert.equal((await loadTransferPack(t, source, some(first.saved), { resolve: { limits: { maxDeltaChain: deepest - 1 } } })).error.code, 'limit_exceeded');
461 assert.equal((await loadTransferPack(t, source, some(first.saved))).error.code, 'limit_exceeded');
462 // limits that do not parse are refused where they enter, saved index or not
463 for (const saved of [none, some(first.saved)]) {
464 const bad = await loadTransferPack(t, source, saved, { resolve: { limits: { maxDeltaChain: -1 } } });
465 assert.equal(bad.ok, false);
466 assert.equal(bad.error.code, 'bad_request');
467 assert.match(bad.error.message, /limits\.maxDeltaChain/u);
468 }
469 // and so are scan.limits, the cap the saved index's entry count is held to (#310 review)
470 for (const saved of [none, some(first.saved)]) {
471 const bad = await loadTransferPack(t, source, saved, { scan: { limits: { maxPackObjects: -1 } } });
472 assert.equal(bad.ok, false);
473 assert.equal(bad.error.code, 'bad_request');
474 assert.match(bad.error.message, /limits\.maxPackObjects/u);
475 }
476 // a direct caller of decodeSavedIndex passes the same limit; without it the default 64 applies
477 const id = { trailer: first.from.pack.trailer, bytes: first.from.pack.bytes };
478 assert.ok(decodeSavedIndex(first.saved, id, 200).ok);
479 assert.equal(decodeSavedIndex(first.saved, id).error.field, 'resolved.depth');
480 // #260 re-review REQ 1: the resolve options are read as resolvePackRange reads them (opts[key]), so one that reaches
481 // the caller's record through a prototype or a class accessor counts. The expected outcomes are not ply's: git reads
482 // this pack as 100 deep (above), so only an honoured limit of 200 resolves it, and a cache of -1 bytes is no size.
483 assert.equal(must(await loadTransferPack(t, source, none, { resolve: Object.create({ limits }) })).loaded.tag, 'scanned');
484 let cacheReads = 0;
485 class Accessors { get limits() { return limits; } get cacheBytes() { cacheReads++; return -1; } }
486 const supplied = [Object.create({ limits, cacheBytes: -1 }), Object.assign(Object.create({ cacheBytes: -1 }), { limits }), new Accessors()];
487 for (const [k, resolve] of supplied.entries()) {
488 const bad = await loadTransferPack(t, source, none, { resolve });
489 assert.equal(bad.ok, false, `inherited cacheBytes -1 (${k}) is refused`);
490 assert.equal(bad.error.code, 'bad_request');
491 assert.match(bad.error.message, /cacheBytes must be a non-negative number, got -1/u);
492 }
493 assert.equal(cacheReads, 1, 'an accessor is read once');
494});
496test('loadTransferPack refusals: a spool too small for a pack, a spool whose read fails, a spool that is not a pack', async () => {
497 const { t, spans } = traced();
498 const tiny = await loadTransferPack(t, rangeFromBytes(new Uint8Array(31)), none);
499 assert.equal(tiny.error.code, 'bad_request');
500 assert.match(tiny.error.message, /holds no pack/);
501 const failing = { size: 1000, read: async () => ({ ok: false, error: Object.assign(new Error('r2 down'), { code: 'network' }) }) };
502 assert.equal((await loadTransferPack(t, failing, none)).error.code, 'network');
503 const short = { size: 1000, read: async () => ({ ok: true, value: streamOf(new Uint8Array(5)) }) };
504 assert.match((await loadTransferPack(t, short, none)).error.message, /read back as 5/);
505 const junk = new Uint8Array(4096); randomFillSync(junk);
506 assert.equal((await loadTransferPack(t, rangeFromBytes(junk), none)).error.code, 'pack_corrupt');
507 assert.equal((await packIdentity(rangeFromBytes(src.packs.ref))).value.bytes, src.packs.ref.length);
508 assert.ok(spans().filter(s => s.name === 'git.transfer.load').every(s => s.status.code === 2), 'each refusal is the span error');
509});
511test('a resumed transfer that loads the saved index every request reaches the fresh head, at every step boundary', async () => {
512 for (const kind of ['ref', 'ofs']) {
513 const bytes = src.packs[kind];
514 const first = must(await loadTransferPack(unwrap(createTransport(`${srv.url}/x.git`)), rangeFromBytes(bytes), none));
515 const plan = must(planTransfer(first.from.pack, first.from.resolved, { budget: BUDGET, ref: REF, commit: src.commit }));
516 assert.ok(plan.steps.length >= 3, `${kind}: ${plan.steps.length} steps`);
517 const name = `s${n++}`;
518 const bare = makeBareRepo(root, name);
519 const t = unwrap(createTransport(`${srv.url}/${name}.git`));
520 let saved = first.saved;
521 let requests = 0;
522 for (;;) {
523 // a new request: nothing but the plan, the spool and the saved bytes
524 const spool = counted(bytes);
525 const loaded = must(await loadTransferPack(t, spool.source, some(saved)));
526 assert.equal(loaded.loaded.tag, 'saved');
527 assert.deepEqual(spool.c, { reads: 1, bytes: 20 });
528 saved = loaded.saved;
529 const r = must(await runTransfer(t, loaded.from, plan, src.commit, { maxSteps: 1, ref: REF }));
530 requests++;
531 if (r.tag === 'done') break;
532 }
533 assert.equal(requests, plan.steps.length);
534 assert.equal(git(bare, ['rev-parse', REF]).trim(), plan.head);
535 assert.equal(git(bare, ['for-each-ref', 'refs/ply']).trim(), '');
536 assert.deepEqual(fsckProblems(bare, { args: ['--strict'] }), []);
537 assert.equal(git(bare, ['ls-tree', '-r', REF]), src.lsTree);
538 // the same plan from a fresh scan: the same head
539 const scanned = await fresh(bytes);
540 assert.equal(must(planTransfer(scanned.pack, scanned.resolved, { budget: BUDGET, ref: REF, commit: src.commit })).head, plan.head);
541 }
542});
544// The Abort contract (README) for packIdentity, whose one read is the spool's last 20 bytes: aborted before the call
545// (nothing asked of the storage), while that read never opens, and while its range is open and never serves a byte.
546// Expected values are the contract's: the code `aborted`, and on the test's own counters no read for the first, the one
547// read told through its signal for the others, and the open range cancelled.
548test('packIdentity aborted before the call, at the trailer read, and in the trailer range settles aborted', async () => {
549 const bytes = src.packs.ofs;
550 const seen = {};
551 const before = stallAt(bytes, 0, 'never');
552 seen.before = { code: codeOf(await packIdentity(before, abortedBefore())), ...before.state };
553 for (const how of ['never', 'stall']) {
554 const source = stallAt(bytes, 0, how);
555 const ac = new AbortController();
556 const op = packIdentity(source, ac.signal);
557 assert.equal(await settledWithin(source.parked), 'settled', how);
558 assert.equal(await settledWithin(op, 20), 'pending', how);
559 ac.abort(new Error('stop'));
560 assert.equal(await settledWithin(op), 'settled', how);
561 seen[how] = { code: codeOf(await op), ...source.state };
562 }
563 assert.deepEqual(seen, {
564 before: { code: 'aborted', reads: 0, cancelled: 0, aborted: 0 },
565 never: { code: 'aborted', reads: 1, cancelled: 0, aborted: 1 },
566 stall: { code: 'aborted', reads: 1, cancelled: 1, aborted: 1 },
567 });
568});
570// #299 item 5: a saved index is held to maxPackObjects, the cap a scan holds the pack's header count to, from its
571// header, before any column is built: one entry over is limit_exceeded (got = the header's count, max = the limit),
572// at the bound it decodes. The count comes from git (rev-list --objects of the packed commit), not from ply.
573test('maxPackObjects: a saved index over it is refused from the header with got/max, at the bound decodes, a 1M-entry header refused before its body is read', async () => {
574 const from = await fresh(src.packs.ref);
575 const bytes = must(encodeSavedIndex(from.pack, some(from.resolved)));
576 const n = git(work, ['rev-list', '--objects', '--no-walk', src.head]).trim().split('\n').length;
577 assert.equal(from.pack.count, n);
578 const r = decodeSavedIndex(bytes, idOf(from), 64, n - 1);
579 assert.deepEqual(r, { ok: false, error: { tag: 'limit_exceeded', what: 'pack objects', got: n, max: n - 1 } });
580 assert.equal(describeRefusal(r.error), `saved index has ${n} entries by its header, over limits.maxPackObjects ${n - 1}`);
581 assertSame(from, must(decodeSavedIndex(bytes, idOf(from), 64, n)));
582 // a header claiming 1,000,000 entries (flags 1: oids) and nothing after it: refused by count, not truncated
583 const head = Uint8Array.from(bytes.subarray(0, 84));
584 const v = new DataView(head.buffer);
585 v.setUint16(6, 1, true);
586 v.setUint32(16, 1_000_000, true);
587 assert.deepEqual(decodeSavedIndex(head, idOf(from), 64, 100_000), { ok: false, error: { tag: 'limit_exceeded', what: 'pack objects', got: 1_000_000, max: 100_000 } });
588 // the default bound is DEFAULT_LIMITS.maxPackObjects, 100 000 (README): the same header is refused, 100 000 is truncated
589 assert.deepEqual(decodeSavedIndex(head, idOf(from)).error, { tag: 'limit_exceeded', what: 'pack objects', got: 1_000_000, max: 100_000 });
590 v.setUint32(16, 100_000, true);
591 assert.equal(decodeSavedIndex(head, idOf(from)).error.tag, 'truncated');
592});
594// The bound on the consumer path: the cap is the scan's (opts.scan.limits.maxPackObjects), and a saved index over it
595// is refused like any other stored bytes and rescanned (README: stored bytes are untrusted, loadTransferPack then scans
596// again); a pack really over the cap is refused by the scan's own header-count check.
597test('loadTransferPack: a saved index over the scan\'s maxPackObjects is refused and rescanned; the scan refuses a pack over it', async () => {
598 const { t } = traced();
599 const first = must(await loadTransferPack(t, rangeFromBytes(src.packs.ref), none));
600 const n = git(work, ['rev-list', '--objects', '--no-walk', src.head]).trim().split('\n').length;
601 assert.equal(first.from.pack.count, n);
602 const over = counted(src.packs.ref);
603 const r = await loadTransferPack(t, over.source, some(first.saved), { scan: { limits: { maxPackObjects: n - 1 } } });
604 assert.equal(r.ok, false);
605 assert.equal(r.error.code, 'limit_exceeded');
606 assert.ok(over.c.bytes > 20, `rescanned: ${over.c.bytes} B read`);
607 // a resolve limit is not the cap: the index is used
608 const at = must(await loadTransferPack(t, rangeFromBytes(src.packs.ref), some(first.saved), { resolve: { limits: { maxPackObjects: n - 1 } } }));
609 assert.deepEqual(at.loaded, { tag: 'saved' });
610});
612// A corrupt stored count (bit 0x02 of byte 18, the third byte of the little-endian u32 count at 16, adds 0x20000) is
613// untrusted data: refused, rescanned, and the load succeeds.
614test('loadTransferPack: a saved index whose stored count is corrupt past the cap is rescanned, with the refusal', async () => {
615 const { t, spans } = traced();
616 const first = must(await loadTransferPack(t, rangeFromBytes(src.packs.ref), none));
617 const n = git(work, ['rev-list', '--objects', '--no-walk', src.head]).trim().split('\n').length;
618 const bad = Uint8Array.from(first.saved);
619 assert.equal(bad[18] & 0x02, 0);
620 bad[18] ^= 0x02;
621 const r = must(await loadTransferPack(t, rangeFromBytes(src.packs.ref), some(bad)));
622 assert.deepEqual(r.loaded, { tag: 'scanned', refused: some({ tag: 'limit_exceeded', what: 'pack objects', got: n + 0x20000, max: 100_000 }) });
623 assert.equal(r.from.pack.count, n);
624 assert.deepEqual(r.saved, first.saved);
625 const span = spans().filter(s => s.name === 'git.transfer.load').at(-1);
626 assert.equal(span.attrs['git.transfer.load.refused'], 'limit_exceeded');
627});
629// Raised scan cap: a pack of 100 005 blobs (git pack-objects) scans under scan.limits.maxPackObjects 200 000 and
630// resumes from its own saved bytes with the same opts.
631test('loadTransferPack: a pack over the default cap scanned under a raised scan cap resumes from its saved bytes', async () => {
632 const dir = join(root, 'big');
633 initRepo(dir);
634 const N = 100_005;
635 let fi = '';
636 for (let i = 0; i < N; i++) { const d = `b${i}\n`; fi += `blob\ndata ${d.length}\n${d}`; }
637 gitBuf(dir, ['fast-import', '--quiet', '--export-marks=marks'], Buffer.from(fi));
638 const oids = git(dir, ['cat-file', '--batch-all-objects', '--batch-check=%(objectname)'], { maxBuffer: 1 << 26 });
639 assert.equal(oids.trim().split('\n').length, N);
640 const pack = new Uint8Array(gitBuf(dir, ['pack-objects', '--stdout', '-q'], Buffer.from(oids)));
641 const { t } = traced();
642 const opts = { scan: { limits: { maxPackObjects: 200_000 } } };
643 const first = must(await loadTransferPack(t, rangeFromBytes(pack), none, opts));
644 assert.equal(first.from.pack.count, N);
645 const again = counted(pack);
646 const r = must(await loadTransferPack(t, again.source, some(first.saved), opts));
647 assert.deepEqual(r.loaded, { tag: 'saved' });
648 assert.deepEqual(again.c, { reads: 1, bytes: 20 });
649}, 120_000);
651// #299 items 5 and 6: decoded columns are views into the saved bytes (one copy, not two), the same for a Node Buffer
652// and a Uint8Array (README 'Saved pack index'): a byte the caller changes after the decode shows in the column.
653test('decodeSavedIndex: columns view the saved bytes, a Buffer and a Uint8Array alike; number columns too', async () => {
654 const from = await fresh(src.packs.ref);
655 const bytes = must(encodeSavedIndex(from.pack, some(from.resolved)));
656 for (const input of [Uint8Array.from(bytes), Buffer.from(bytes)]) {
657 let views = 0;
658 const back = must(decodeSavedIndex(input, idOf(from)));
659 assertSame(from, back);
660 assertSameResolved(from.resolved, back.resolved.value);
661 for (const k of [...COLS.filter(c => c !== 'baseIndex'), 'baseIndex']) {
662 const col = back.pack.columns[k];
663 if (col.length === 0) continue;
664 if (col.BYTES_PER_ELEMENT > 1) views++;
665 assert.equal(col.buffer, input.buffer, k);
666 const at = col.byteOffset - input.byteOffset;
667 const was = col[0];
668 input[at] ^= 0xff;
669 assert.notEqual(col[0], was, `${k} sees a write to the saved bytes`);
670 input[at] ^= 0xff;
671 assert.equal(col[0], was, k);
672 }
673 // base is rebuilt from the pack's baseIndex on decode, not read from the bytes; the others are views
674 for (const k of RCOLS.filter(c => c !== 'base')) {
675 const col = back.resolved.value.columns[k];
676 // a number column is a view only where it is aligned in the buffer (README); here the resolved section follows
677 // byte columns of odd length, so a misaligned one must be a copy with the same values
678 if (col.BYTES_PER_ELEMENT > 1 && col.buffer !== input.buffer) { assert.notEqual(col.buffer, input.buffer); continue; }
679 if (col.BYTES_PER_ELEMENT > 1) views++;
680 assert.equal(col.buffer, input.buffer, k);
681 const at = col.byteOffset - input.byteOffset;
682 const was = col[0];
683 input[at] ^= 0xff;
684 assert.notEqual(col[0], was, `resolved ${k} sees a write to the saved bytes`);
685 input[at] ^= 0xff;
686 }
687 // the aligned path is taken: number columns (offset, compressed, size, baseIndex) are views, not copies
688 assert.ok(views >= 4, `${views} number columns are views`);
689 }
690});
Diagnostics6 git calls · 234 ms
CallStatusSizeTime0234 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB93 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=93
git.response.bytes=11990
git.response.cache=EXPIRED
git.response.duration_ms=93
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch20013 KiB133 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=133
git.response.bytes=13674
git.response.cache=MISS
git.response.duration_ms=133
git.fetch.wants=5ecf8696e353b1c7bd95bbed35bc5280c7fed854
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=5
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=13674
git.pkt.payload.max=8192
git.sideband.band1.bytes=13642
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e1827e38b63014
Request
6fe3decc…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format