main
1// planPackBatches (#237): a spooled pack's entries split into pushes under a caller's budget, children first.
2// - deterministic, plain data (a JSON round trip keeps it and its hash), hash stable and moved by the budget
3// - every batch within every budget dimension; every per-object check made before any batch
4// - children first, against the trees themselves: every blob a tree names is in a batch no later than the tree's,
5// and every commit's tree no later than the commit's; within a batch every base before its delta
6// - delta families kept whole, or split in chain order with REF rewrites under crossBatchRef
7// - refusals: per-object limits, a family too big without crossBatchRef, trees that do not fit one push, a base
8// outside the selection, an unresolved entry, a bad selection or budget
9// - end to end against git http-backend with receive.fsckObjects: each batch packed (git pack-objects over exactly
10// the batch's oids, standing in for packSubset #235) with a generated temp tree and commit chained per batch,
11// pushed with pushPack, the final push atomic (create main, delete the temp ref); fsck --strict, every object
12// - planning time and memory at 200,000 entries
13import { test, beforeAll as before, afterAll as after } from 'vitest';
14import assert from 'node:assert/strict';
15import { mkdirSync, writeFileSync } from 'node:fs';
16import { join } from 'node:path';
17import { randomFillSync, createHash } from 'node:crypto';
18import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, fsckProblems, initRepo } from './server.js';
19import { scanPackIndex } from '../src/packscanindex.js';
20import { resolvePackRange, NO_BASE, THIN_BASE } from '../src/packrangeresolve.js';
21import { rangeFromBytes } from '../src/rangesource.js';
22import { planPackBatches, planHash, PLAN_VERSION } from '../src/packplan.js';
23import { resolvePack, parsePack } from '../src/pack.js';
24import { parseTree } from '../src/objects.js';
25import { createTransport } from '../src/http.js';
26import { pushPack } from '../src/receivecaps.js';
27import { ZERO_OID } from '../src/push.js';
28import { TYPE } from '../src/objects.js';
29import { unwrap } from './unwrap.js';
31const must = r => { if (r.ok) return r.value; throw Object.assign(new Error(r.error.message), { code: r.error.code }); };
32const streamOf = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
33const ident = ['-c', 'user.name=a', '-c', 'user.email=a@x'];
34const BIG = { maxRequestBytes: 1 << 30, maxObjects: 1 << 20, maxObjectBytes: 1 << 30, maxCompressedObjectBytes: 1 << 30, reserveBytes: 0, reserveObjects: 0 };
36let root, cleanup, srv, n = 0, work, packs;
37// A source repository: incompressible blobs (so sizes are what they look like), a blob edited over several commits
38// (a delta chain), nested trees; packed REF-only (as Artifacts sends) and with OFS deltas.
39before(async () => {
40 ({ root, cleanup } = tmpRoot());
41 srv = await startGitServer(root);
42 work = join(root, 'source');
43 mkdirSync(join(work, 'a', 'b'), { recursive: true });
44 initRepo(work);
45 for (let i = 0; i < 12; i++) {
46 const blob = new Uint8Array(40_000 + i * 1000);
47 randomFillSync(blob);
48 writeFileSync(join(work, i % 3 === 0 ? 'a' : i % 3 === 1 ? join('a', 'b') : '.', `r${i}.bin`), blob);
49 }
50 const base = new Uint8Array(60_000);
51 randomFillSync(base);
52 for (let v = 0; v < 6; v++) {
53 base[v * 997] ^= 0xff;
54 writeFileSync(join(work, 'a', 'chain.bin'), base);
55 writeFileSync(join(work, 'small.txt'), `version ${v}\n`.repeat(200));
56 git(work, ['add', '.']);
57 git(work, [...ident, 'commit', '-q', '-m', `v${v}`]);
58 }
59 const head = git(work, ['rev-parse', 'HEAD']).trim();
60 const objs = git(work, ['rev-list', '--objects', head]);
61 packs = {
62 head,
63 ref: new Uint8Array(gitBuf(work, ['pack-objects', '--stdout', '-q', '--depth=50'], objs)),
64 ofs: new Uint8Array(gitBuf(work, ['pack-objects', '--stdout', '-q', '--depth=50', '--delta-base-offset'], objs)),
65 };
66});
67after(async () => { await srv?.close(); cleanup?.(); });
69async function indexed(pack) {
70 const idx = must(await scanPackIndex(streamOf(pack)));
71 const res = must(await resolvePackRange(idx, rangeFromBytes(pack)));
72 return { idx, res };
73}
74const hexOf = (res, i) => res.oid(i).value;
76// Every batch within the budget; every entry planned once; within a batch, every in-batch base before its delta.
77function assertWithin(plan, idx, res, budget) {
78 const seen = new Map();
79 plan.batches.forEach((b, bi) => {
80 assert.ok(b.packBytes + budget.reserveBytes <= budget.maxRequestBytes, `batch ${bi} bytes ${b.packBytes}`);
81 assert.ok(b.objects + budget.reserveObjects <= budget.maxObjects, `batch ${bi} objects ${b.objects}`);
82 assert.equal(b.objects, b.entries.length);
83 b.entries.forEach((i, k) => { assert.equal(seen.has(i), false, `entry ${i} twice`); seen.set(i, { batch: bi, at: k }); });
84 });
85 for (const [i, where] of seen) {
86 assert.ok(res.columns.size[i] <= budget.maxObjectBytes);
87 assert.ok(idx.columns.compressed[i] <= budget.maxCompressedObjectBytes);
88 const b = res.columns.base[i];
89 if (b >= 0 && seen.has(b)) {
90 const wb = seen.get(b);
91 assert.ok(wb.batch < where.batch || (wb.batch === where.batch && wb.at < where.at), `base ${b} of ${i} not before it`);
92 }
93 }
94 return seen;
95}
96// Children first, read from the objects: every tree's blobs and subtrees, and every commit's tree, in a batch no later.
97function assertChildrenFirst(plan, pack, res, seen) {
98 const objects = must(resolvePack(must(parsePack(pack)))).objects;
99 const batchOfOid = new Map();
100 for (const [i, w] of seen) batchOfOid.set(hexOf(res, i), w.batch);
101 let edges = 0;
102 for (const [oid, o] of objects) {
103 const mine = batchOfOid.get(oid);
104 if (mine === undefined) continue;
105 const kids = o.type === TYPE.TREE ? must(parseTree(o.data)).map(e => e.oid)
106 : o.type === TYPE.COMMIT ? [/^tree ([0-9a-f]{40})/.exec(new TextDecoder().decode(o.data))[1]] : [];
107 for (const k of kids) {
108 const theirs = batchOfOid.get(k);
109 if (theirs === undefined) continue;
110 edges++;
111 assert.ok(theirs <= mine, `${oid} (batch ${mine}) names ${k} (batch ${theirs})`);
112 }
113 }
114 return edges;
115}
117test('REF-only and OFS packs: every batch within every budget dimension, children first, families whole, deterministic', async () => {
118 for (const kind of ['ref', 'ofs']) {
119 const { idx, res } = await indexed(packs[kind]);
120 const budget = { maxRequestBytes: 160_000, maxObjects: 40, maxObjectBytes: 1 << 20, maxCompressedObjectBytes: 1 << 20, reserveBytes: 2_000, reserveObjects: 2 };
121 const plan = must(planPackBatches(idx, res, { budget }));
122 assert.ok(plan.batches.length >= 3, `${kind}: ${plan.batches.length} batches`);
123 assert.equal(plan.entries, idx.count);
124 assert.equal(plan.version, PLAN_VERSION);
125 assert.equal(plan.trailer, idx.trailer);
126 assert.deepEqual(plan.batches.slice(0, -1).map(b => b.kind), plan.batches.slice(0, -1).map(() => 'blobs'));
127 assert.equal(plan.batches.at(-1).kind, 'final');
128 for (const b of plan.batches.slice(0, -1)) for (const i of b.entries) assert.equal(res.columns.type[i], TYPE.BLOB);
129 const seen = assertWithin(plan, idx, res, budget);
130 assert.equal(seen.size, idx.count);
131 assert.ok(assertChildrenFirst(plan, packs[kind], res, seen) > 10);
132 // families whole: no entry's in-pack base is in another batch
133 for (const b of plan.batches) assert.equal(b.refRewrites, 0);
134 for (const [i, w] of seen) { const base = res.columns.base[i]; if (base >= 0) assert.equal(seen.get(base).batch, w.batch); }
135 // deterministic, plain data
136 const again = must(planPackBatches(idx, res, { budget }));
137 assert.deepEqual(again, plan);
138 const round = JSON.parse(JSON.stringify(plan));
139 assert.deepEqual(round, plan);
140 const { hash, ...body } = round;
141 assert.equal(planHash(body), hash);
142 assert.match(hash, /^[0-9a-f]{40}$/);
143 }
144});
146test('the hash names the plan: stable across runs, moved by the budget, the options and the source pack', async () => {
147 const { idx, res } = await indexed(packs.ref);
148 const budget = { maxRequestBytes: 160_000, maxObjects: 40, maxObjectBytes: 1 << 20, maxCompressedObjectBytes: 1 << 20, reserveBytes: 2_000, reserveObjects: 2 };
149 const h = must(planPackBatches(idx, res, { budget })).hash;
150 assert.equal(must(planPackBatches(idx, res, { budget })).hash, h);
151 const moved = new Set([h]);
152 for (const k of Object.keys(budget)) moved.add(must(planPackBatches(idx, res, { budget: { ...budget, [k]: budget[k] + 1 } })).hash);
153 assert.equal(moved.size, 1 + Object.keys(budget).length, 'each budget field moves the hash');
154 assert.notEqual(must(planPackBatches(idx, res, { budget, crossBatchRef: true })).hash, h);
155 assert.notEqual(must(planPackBatches(idx, res, { budget, select: [...Array(idx.count).keys()] })).hash, h);
156 const ofs = await indexed(packs.ofs);
157 assert.notEqual(must(planPackBatches(ofs.idx, ofs.res, { budget })).hash, h);
158 // and the canonical form is what is hashed: a tampered plan no longer matches
159 const plan = must(planPackBatches(idx, res, { budget }));
160 const { hash, ...body } = plan;
161 assert.notEqual(planHash({ ...body, batches: [...body.batches].reverse() }), hash);
162 assert.notEqual(planHash({ ...body, trailer: '0'.repeat(40) }), hash);
163});
165test('one big batch when everything fits; blobs merge into the final batch when they fit beside the trees', async () => {
166 const { idx, res } = await indexed(packs.ref);
167 const plan = must(planPackBatches(idx, res, { budget: BIG }));
168 assert.equal(plan.batches.length, 1);
169 assert.equal(plan.batches[0].kind, 'final');
170 assert.equal(plan.batches[0].objects, idx.count);
171 const packBytes = idx.columns.compressed.reduce((s, c, i) => s + c + idx.columns.headerBytes[i], 0) + 32;
172 assert.equal(plan.batches[0].packBytes, packBytes, 'REF entries are counted as stored');
173 // only blobs selected: no final batch
174 const blobs = [...Array(idx.count).keys()].filter(i => res.columns.type[i] === TYPE.BLOB);
175 const p2 = must(planPackBatches(idx, res, { budget: BIG, select: blobs }));
176 assert.deepEqual(p2.batches.map(b => b.kind), ['blobs']);
177 // nothing selected: no batches
178 assert.deepEqual(must(planPackBatches(idx, res, { budget: BIG, select: [] })).batches, []);
179});
181test('OFS distances are counted at the length the largest distance in a push needs (header growth)', async () => {
182 const { idx, res } = await indexed(packs.ofs);
183 const ofs = [...Array(idx.count).keys()].filter(i => idx.columns.storedType[i] === TYPE.OFS_DELTA);
184 assert.ok(ofs.length > 3);
185 const plan = must(planPackBatches(idx, res, { budget: BIG }));
186 const stored = idx.columns.compressed.reduce((s, c, i) => s + c + idx.columns.headerBytes[i], 0) + 32;
187 // maxRequestBytes 2^30 needs a 5-byte distance; every OFS entry counts that, whatever its stored distance was
188 let grown = 0;
189 for (const i of ofs) {
190 const d = idx.columns.offset[i] - idx.columns.offset[idx.columns.baseIndex[i]];
191 let len = 1;
192 for (let r = Math.floor(d / 128); r > 0; r = Math.floor((r - 1) / 128)) len++;
193 grown += 5 - len;
194 }
195 assert.equal(plan.batches[0].packBytes, stored + grown);
196});
198test('a family too big for one push: refused, or split in chain order with REF rewrites under crossBatchRef', async () => {
199 for (const kind of ['ref', 'ofs']) {
200 const { idx, res } = await indexed(packs[kind]);
201 // the chain.bin family is the deepest; size a push to hold its root but not the whole family
202 let deepest = 0;
203 for (let i = 0; i < idx.count; i++) if (res.columns.depth[i] > res.columns.depth[deepest]) deepest = i;
204 assert.ok(res.columns.depth[deepest] >= 2, `${kind}: chain depth ${res.columns.depth[deepest]}`);
205 let r = deepest;
206 while (res.columns.base[r] >= 0) r = res.columns.base[r];
207 const family = [...Array(idx.count).keys()].filter(i => { let u = i; while (res.columns.base[u] >= 0) u = res.columns.base[u]; return u === r; });
208 assert.ok(family.length >= 4, `${kind}: family of ${family.length}`);
209 // counted at the OFS length a push of about this size needs (the count depends on maxRequestBytes)
210 const big = must(planPackBatches(idx, res, { budget: BIG, select: family })).batches[0].packBytes;
211 const whole = must(planPackBatches(idx, res, { budget: { ...BIG, maxRequestBytes: big }, select: family })).batches[0].packBytes;
212 const budget = { ...BIG, maxObjects: family.length, maxRequestBytes: whole - 1 };
213 const select = family;
214 const refused = planPackBatches(idx, res, { budget, select });
215 assert.equal(refused.ok, false);
216 assert.equal(refused.error.code, 'limit_exceeded');
217 assert.match(refused.error.message, /delta family bytes/);
218 const plan = must(planPackBatches(idx, res, { budget, select, crossBatchRef: true }));
219 assert.ok(plan.batches.length >= 2, `${kind}: ${plan.batches.length}`);
220 assert.ok(plan.batches.slice(1).some(b => b.refRewrites > 0));
221 assertWithin(plan, idx, res, budget);
222 // a family over the object budget is refused by objects
223 const byObjects = planPackBatches(idx, res, { budget: { ...BIG, maxObjects: family.length - 1 }, select });
224 assert.equal(byObjects.ok, false);
225 assert.match(byObjects.error.message, /delta family objects/);
226 const splitByObjects = must(planPackBatches(idx, res, { budget: { ...BIG, maxObjects: 2 }, select, crossBatchRef: true }));
227 assert.ok(splitByObjects.batches.every(b => b.objects <= 2));
228 assert.equal(splitByObjects.batches.reduce((s, b) => s + b.objects, 0), family.length);
229 }
230});
232test('refusals before any batch: per-object limits, an entry no push holds, trees over one push, bases outside the selection', async () => {
233 const { idx, res } = await indexed(packs.ref);
234 const biggest = Math.max(...res.columns.size);
235 const big = planPackBatches(idx, res, { budget: { ...BIG, maxObjectBytes: biggest - 1 } });
236 assert.equal(big.error.code, 'limit_exceeded');
237 assert.match(big.error.message, /object bytes/);
238 assert.equal(big.error.got, biggest);
239 assert.equal(big.error.max, biggest - 1);
240 assert.ok(must(planPackBatches(idx, res, { budget: { ...BIG, maxObjectBytes: biggest } })));
241 const comp = Math.max(...idx.columns.compressed);
242 const c = planPackBatches(idx, res, { budget: { ...BIG, maxCompressedObjectBytes: comp - 1 } });
243 assert.match(c.error.message, /compressed object bytes/);
244 assert.equal(c.error.got, comp);
245 assert.ok(must(planPackBatches(idx, res, { budget: { ...BIG, maxCompressedObjectBytes: comp } })));
246 // trees, commits and tags must fit one push
247 const rest = [...Array(idx.count).keys()].filter(i => res.columns.type[i] !== TYPE.BLOB);
248 const restBytes = rest.reduce((s, i) => s + idx.columns.compressed[i] + idx.columns.headerBytes[i], 32);
249 const trees = planPackBatches(idx, res, { budget: { ...BIG, maxRequestBytes: restBytes - 1 }, select: rest });
250 assert.equal(trees.error.code, 'limit_exceeded');
251 assert.match(trees.error.message, /tree batch bytes/);
252 assert.equal(trees.error.got, restBytes);
253 assert.ok(must(planPackBatches(idx, res, { budget: { ...BIG, maxRequestBytes: restBytes }, select: rest })));
254 const treeObjs = planPackBatches(idx, res, { budget: { ...BIG, maxObjects: rest.length }, select: rest, crossBatchRef: false });
255 assert.ok(treeObjs.ok);
256 const treeObjs2 = planPackBatches(idx, res, { budget: { ...BIG, maxObjects: rest.length, reserveObjects: 1 }, select: rest });
257 assert.match(treeObjs2.error.message, /tree batch objects/);
258 assert.equal(treeObjs2.error.got, rest.length + 1);
259 // an entry no push holds even alone: crossBatchRef splits families, never one object
260 const wholeBlobs = [...Array(idx.count).keys()].filter(i => res.columns.type[i] === TYPE.BLOB && res.columns.base[i] === NO_BASE);
261 const fat = wholeBlobs.reduce((a, i) => (idx.columns.compressed[i] > idx.columns.compressed[a] ? i : a), wholeBlobs[0]);
262 const fatBytes = 32 + idx.columns.headerBytes[fat] + idx.columns.compressed[fat];
263 const noPush = planPackBatches(idx, res, { budget: { ...BIG, maxRequestBytes: fatBytes - 1 }, select: [fat], crossBatchRef: true });
264 assert.equal(noPush.error.code, 'limit_exceeded');
265 assert.match(noPush.error.message, /entry bytes \(entry \d+\)/);
266 assert.equal(noPush.error.got, fatBytes);
267 assert.ok(must(planPackBatches(idx, res, { budget: { ...BIG, maxRequestBytes: fatBytes }, select: [fat], crossBatchRef: true })));
268 // a delta selected without its base needs crossBatchRef
269 const delta = [...Array(idx.count).keys()].find(i => res.columns.base[i] >= 0);
270 const lone = planPackBatches(idx, res, { budget: BIG, select: [delta] });
271 assert.equal(lone.error.code, 'bad_request');
272 assert.match(lone.error.message, /base is not in the selection/);
273 const loneOk = must(planPackBatches(idx, res, { budget: BIG, select: [delta], crossBatchRef: true }));
274 assert.equal(loneOk.batches.at(-1).refRewrites, 1);
275});
277test('refusals of input: selection, budget, an index that does not match, an entry with no resolved type, a thin base', async () => {
278 const { idx, res } = await indexed(packs.ref);
279 for (const select of [[-1], [idx.count], [0.5], [0, 0]]) assert.equal(planPackBatches(idx, res, { budget: BIG, select }).error.code, 'bad_request', JSON.stringify(select));
280 for (const k of Object.keys(BIG)) for (const v of [-1, 1.5, Number.NaN, 2 ** 60]) {
281 assert.equal(planPackBatches(idx, res, { budget: { ...BIG, [k]: v } }).error.code, 'bad_request', `${k}=${v}`);
282 }
283 assert.match(planPackBatches(idx, res, { budget: { ...BIG, maxObjects: 3, reserveObjects: 3 } }).error.message, /reserveObjects/);
284 assert.ok(planPackBatches(idx, res, { budget: { ...BIG, maxObjects: 3, reserveObjects: 2 }, select: [] }).ok);
285 assert.match(planPackBatches(idx, res, { budget: { ...BIG, maxRequestBytes: 132, reserveBytes: 100 } }).error.message, /reserveBytes/);
286 assert.ok(planPackBatches(idx, res, { budget: { ...BIG, maxRequestBytes: 133, reserveBytes: 100 }, select: [] }).ok);
287 assert.match(planPackBatches(idx, { ...res, count: res.count - 1 }, { budget: BIG }).error.message, /resolved index has/);
288 const untyped = { ...res, columns: { ...res.columns, type: Uint8Array.from(res.columns.type, (t, i) => (i === 3 ? 0 : t)) } };
289 assert.match(planPackBatches(idx, untyped, { budget: BIG }).error.message, /entry 3 has no resolved type/);
290 assert.ok(planPackBatches(idx, untyped, { budget: BIG, select: [0, 1, 2].filter(i => res.columns.base[i] < 0) }).ok);
291 const delta = [...Array(idx.count).keys()].find(i => res.columns.base[i] >= 0);
292 const thin = { ...res, columns: { ...res.columns, base: Int32Array.from(res.columns.base, (b, i) => (i === delta ? THIN_BASE : b)) } };
293 assert.match(planPackBatches(idx, thin, { budget: BIG }).error.message, /base is not in the selection/);
294 const thinPlan = must(planPackBatches(idx, thin, { budget: BIG, crossBatchRef: true }));
295 assert.equal(thinPlan.batches[0].refRewrites, 1);
296 assert.equal(NO_BASE, -1);
297});
299test('end to end: each batch pushed with pushPack to git http-backend (receive.fsckObjects), final push atomic; fsck --strict, every object', async () => {
300 for (const kind of ['ref', 'ofs']) {
301 const name = `e2e${n++}`;
302 const bare = makeBareRepo(root, name);
303 const url = `${srv.url}/${name}.git`;
304 const { idx, res } = await indexed(packs[kind]);
305 const budget = { maxRequestBytes: 160_000, maxObjects: 40, maxObjectBytes: 1 << 20, maxCompressedObjectBytes: 1 << 20, reserveBytes: 8_000, reserveObjects: 2 };
306 const plan = must(planPackBatches(idx, res, { budget }));
307 assert.ok(plan.batches.length >= 3);
308 const tempRef = `refs/ply/transfer/${plan.hash}`;
309 const t = unwrap(createTransport(url));
310 let prevTemp = ZERO_OID;
311 for (const [bi, b] of plan.batches.entries()) {
312 const oids = b.entries.map(i => hexOf(res, i));
313 const commands = [];
314 let extra = [];
315 if (b.kind === 'blobs') {
316 // the generated temp tree (this batch's blobs, named by oid) and commit, chained on the last one
317 const tree = git(work, ['mktree'], { input: oids.map(o => `100644 blob ${o}\t${o}\n`).join('') }).trim();
318 const commit = git(work, [...ident, 'commit-tree', tree, ...(prevTemp === ZERO_OID ? [] : ['-p', prevTemp]), '-m', `batch ${bi} of ${plan.hash}`]).trim();
319 extra = [tree, commit];
320 commands.push({ ref: tempRef, oldOid: prevTemp, newOid: commit });
321 prevTemp = commit;
322 } else {
323 commands.push({ ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: packs.head });
324 if (prevTemp !== ZERO_OID) commands.push({ ref: tempRef, oldOid: prevTemp, newOid: ZERO_OID });
325 }
326 // stand-in for packSubset (#235): exactly these objects, deltas only among them (or, under fsck, the temp chain)
327 const pack = new Uint8Array(gitBuf(work, ['pack-objects', '--stdout', '-q', ...(kind === 'ofs' ? ['--delta-base-offset'] : [])], [...oids, ...extra].join('\n') + '\n'));
328 assert.ok(pack.length + budget.reserveBytes <= budget.maxRequestBytes + 4_000, `batch ${bi}: ${pack.length}`);
329 const r = await pushPack(t, { commands, pack: streamOf(pack), length: pack.length, atomic: b.kind === 'final' });
330 assert.ok(r.ok, `${kind} batch ${bi}: ${r.ok ? '' : r.error.message}`);
331 }
332 assert.equal(git(bare, ['rev-parse', 'refs/heads/main']).trim(), packs.head);
333 assert.equal(git(bare, ['for-each-ref', 'refs/ply']).trim(), '');
334 assert.deepEqual(fsckProblems(bare, { args: ['--strict'] }), []);
335 const want = git(work, ['rev-list', '--objects', packs.head]).split('\n').filter(Boolean).map(l => l.slice(0, 40)).sort();
336 const got = git(bare, ['rev-list', '--objects', 'refs/heads/main']).split('\n').filter(Boolean).map(l => l.slice(0, 40)).sort();
337 assert.deepEqual(got, want);
338 }
339});
341// A synthetic index of `n` entries: blob families of a root and a chain of deltas, a tree per 50 blobs, one commit.
342// `stored` is how the deltas are stored: REF_DELTA (a 23-byte header: 3 of type and size, a 20-byte id) or OFS_DELTA (4:
343// 2 of type and size for a 40-byte delta, and a 2-byte distance, since the entry before is 203..383 bytes away).
344function synthetic(count, stored = TYPE.REF_DELTA) {
345 const offset = new Float64Array(count), headerBytes = new Uint8Array(count).fill(3), compressed = new Float64Array(count), size = new Float64Array(count);
346 const storedType = new Uint8Array(count), type = new Uint8Array(count), baseIndex = new Int32Array(count).fill(-1);
347 const rbase = new Int32Array(count).fill(-1), depth = new Int32Array(count), rsize = new Float64Array(count);
348 let at = 12;
349 for (let i = 0; i < count; i++) {
350 const isCommit = i === count - 1;
351 const isTree = !isCommit && i % 50 === 49;
352 const famPos = i % 5;
353 const delta = !isCommit && !isTree && famPos > 0 && i % 50 !== 0;
354 offset[i] = at;
355 compressed[i] = 200 + (i % 17) * 10;
356 size[i] = delta ? 40 : 400;
357 rsize[i] = 400;
358 storedType[i] = isCommit ? TYPE.COMMIT : isTree ? TYPE.TREE : delta ? stored : TYPE.BLOB;
359 type[i] = isCommit ? TYPE.COMMIT : isTree ? TYPE.TREE : TYPE.BLOB;
360 if (delta) { rbase[i] = i - 1; depth[i] = depth[i - 1] + 1; headerBytes[i] = stored === TYPE.OFS_DELTA ? 4 : 23; }
361 if (delta && stored === TYPE.OFS_DELTA) baseIndex[i] = i - 1;
362 at += headerBytes[i] + compressed[i];
363 }
364 const idx = { version: 2, count, trailer: createHash('sha1').update(String(count)).digest('hex'), bytes: at + 20, columns: { offset, headerBytes, compressed, size, storedType, type, baseIndex } };
365 const res = { count, columns: { oid: new Uint8Array(count * 20), type, base: rbase, depth, size: rsize } };
366 return { idx, res };
367}
369// Review finding saved-resolved-cycle-oom: planPackBatches is public and takes the caller's columns, so a base column
370// with a cycle must end in a Result. No pack gives one (a delta's base is resolved before it), so every such input is
371// refused; the walk is bounded by the entry count (before the fix this test ended in a heap out-of-memory abort).
372test('a resolved base column with a cycle is refused, not walked forever', () => {
373 for (const [name, cycle] of [['2-cycle', [1, 2]], ['3-cycle', [1, 2, 3]], ['self', [1]], ['the whole family', [1, 2, 3, 4]]]) {
374 const { idx, res } = synthetic(200);
375 // entries 1..4 are REF deltas of one family (synthetic); close a cycle among them, depths left as they were
376 cycle.forEach((i, k) => { res.columns.base[i] = cycle[(k + 1) % cycle.length]; });
377 const r = planPackBatches(idx, res, { budget: BIG, crossBatchRef: true });
378 assert.equal(r.ok, false, name);
379 assert.equal(r.error.code, 'bad_request', name);
380 assert.match(r.error.message, /resolved index is malformed/u, name);
381 }
382 // the same columns without the cycle still plan
383 const { idx, res } = synthetic(200);
384 assert.ok(planPackBatches(idx, res, { budget: BIG }).ok);
385});
387test('planning time and memory at 200,000 entries, REF_DELTA and OFS_DELTA packs', () => {
388 const N = 200_000;
389 const budget = { maxRequestBytes: 64 * 1024 * 1024 - 64 * 1024, maxObjects: 50_000, maxObjectBytes: 32 * 1024 * 1024, maxCompressedObjectBytes: 32 * 1024 * 1024, reserveBytes: 64 * 1024, reserveObjects: 300 };
390 const kinds = [['ref', synthetic(N)], ['ofs', synthetic(N, TYPE.OFS_DELTA)]];
391 const times = { ref: [], ofs: [] };
392 const first = {};
393 // the two kinds alternate, so load on the machine falls on both; the figure reported is each kind's median
394 // The bound on the planner's work is a count the test takes itself, never a time: the ids it asks `resolved` for.
395 // The id is any 40 lowercase hex (the planner checks its form, and names nothing with it).
396 let asked = 0;
397 const counting = res => ({ ...res, oid: () => { asked++; return { tag: 'some', value: 'a'.repeat(40) }; } });
398 for (let round = 0; round < 5; round++) {
399 for (const [kind, { idx, res }] of kinds) {
400 const counted = counting(res);
401 asked = 0;
402 globalThis.gc?.();
403 const before = process.memoryUsage().heapUsed;
404 const t0 = performance.now();
405 const plan = must(planPackBatches(idx, counted, { budget, crossBatchRef: false }));
406 const ms = performance.now() - t0;
407 const after = process.memoryUsage().heapUsed;
408 assert.equal(plan.entries, N);
409 for (const b of plan.batches) assert.ok(b.objects + budget.reserveObjects <= budget.maxObjects && b.packBytes + budget.reserveBytes <= budget.maxRequestBytes);
410 // whole families: nothing is rewritten, whichever way the deltas are stored
411 assert.equal(plan.batches.reduce((sum, b) => sum + b.refRewrites, 0), 0);
412 // by hand: every base is written beside its delta, so no id is written and none is asked for, at any N
413 assert.equal(asked, 0, `${kind}: ids asked of resolved for ${N} entries, families whole`);
414 // printed for README only; nothing is asserted on a time
415 times[kind].push(ms);
416 first[kind] ??= { batches: plan.batches.length, heapDeltaMB: +((after - before) / 1e6).toFixed(1), planJsonMB: +(JSON.stringify(plan).length / 1e6).toFixed(2) };
417 }
418 }
419 const median = a => Math.round(a.toSorted((x, y) => x - y)[Math.floor(a.length / 2)]);
420 for (const [kind] of kinds) console.log(JSON.stringify({ kind, entries: N, medianMs: median(times[kind]), runsMs: times[kind].map(Math.round), ...first[kind] }));
422 // A budget that cuts every family, worked out by hand from `synthetic`. Planned: the blobs alone (so no final batch),
423 // 2 objects a push, crossBatchRef on. Each block of 50 entries is 9 families of 5 (a root and d1..d4, each delta
424 // resting on the entry before it), one family of 4 (entries 45..48) and a tree at 49 (the commit, in the last block),
425 // which is not planned. No family fits a push, so each is split in chain order: [root d1] [d2 d3] [d4], or
426 // [root d1] [d2 d3]. d2 and d4 open a push without their base: 2 cuts in a family of 5 and 1 in a family of 4, so
427 // 9 * 2 + 1 = 19 cuts and 9 * 3 + 2 = 29 pushes a block, and there are N / 50 = 4,000 blocks.
428 const cuts = (N / 50) * 19;
429 assert.equal(cuts, 76_000);
430 const two = { ...budget, maxObjects: 2, reserveObjects: 0 };
431 // A REF_DELTA carries its base's id already: none is asked for. An OFS_DELTA cut from its base needs that id each
432 // time it is counted as a REF_DELTA: once sized in the push it opens, once placed there. Never beside its base.
433 for (const [kind, lookups] of [['ref', 0], ['ofs', 2 * cuts]]) {
434 const { idx, res } = Object.fromEntries(kinds)[kind];
435 const blobs = [...Array(N).keys()].filter(i => res.columns.type[i] === TYPE.BLOB);
436 asked = 0;
437 const plan = must(planPackBatches(idx, counting(res), { budget: two, select: blobs, crossBatchRef: true }));
438 assert.equal(plan.entries, N - N / 50, kind);
439 assert.equal(plan.batches.length, (N / 50) * 29, `${kind}: pushes`);
440 assert.equal(plan.batches.reduce((sum, b) => sum + b.refRewrites, 0), cuts, `${kind}: cuts`);
441 assert.equal(asked, lookups, `${kind}: ids asked of resolved for ${cuts} cut deltas`);
442 }
443});
Diagnostics4 git calls · 384 ms
CallStatusSizeTime0384 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit5 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2008.3 KiB377 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=377 git.response.bytes=8519 git.response.cache=MISS git.response.duration_ms=377 git.fetch.wants=0f756a2e5f9ff0b82ee7f032f3ca09205d76f167 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=8519 git.pkt.payload.max=8498 git.sideband.band1.bytes=8497 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2a8bbdb12e6b0
- Request
- 07570eed…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format