GitHubivarvong
ply
main
1// resolvePack's cost and its stack. A pack's deltas may arrive in any order (a REF_DELTA before its base, an OFS chain
2// resting on a REF_DELTA whose base is later still), so the resolver's work must not depend on the order: every entry is
3// visited a constant number of times, a chain of any configured length is walked without recursion, and the Result is
4// never a thrown RangeError. Oracles: node:crypto for ids, node:zlib for the pack bytes, `git index-pack` plus
5// `git cat-file --batch` for what the pack holds; the step bounds are worked out by hand below.
6import { test } from 'vitest';
7import assert from 'node:assert/strict';
8import { deflateSync } from 'node:zlib';
9import { createHash } from 'node:crypto';
10import { parsePack, resolvePack, describePack } from '../src/packresolve.js';
11import { withLimits } from '../src/limits.js';
12import { gitBuf, git, tmpRoot } from './server.js';
14const BLOB = 3; // gitformat-pack: OBJ_BLOB
15const OFS = 6;
16const REF = 7;
17const sha1 = (type, data) => createHash('sha1').update(`${type} ${data.length}\0`).update(data).digest();
18const content = i => Buffer.from(`c${String(i).padStart(9, '0')}\n`);
19// A delta that copies nothing: the source size, the result size, one insert of the whole result (11 bytes, under 127).
20const replaceDelta = (srcLen, bytes) => Uint8Array.of(srcLen, bytes.length, bytes.length, ...bytes);
21const objHeader = (type, size) => { let b = (type << 4) | (size & 15); let rest = size >> 4; const out = []; while (rest) { out.push(b | 0x80); b = rest & 0x7f; rest >>= 7; } out.push(b); return out; };
22const ofsDistance = d => { const out = [d & 0x7f]; let rest = d >> 7; while (rest) { rest -= 1; out.unshift(0x80 | (rest & 0x7f)); rest >>= 7; } return out; };
23class Pack {
24 parts = []; at = 12; n = 0;
25 add(type, data, base = {}) {
26 const offset = this.at;
27 const head = type === OFS ? [...objHeader(OFS, data.length), ...ofsDistance(offset - base.ofs)] : type === REF ? [...objHeader(REF, data.length), ...base.ref] : objHeader(type, data.length);
28 const b = Buffer.concat([Buffer.from(head), deflateSync(data, { level: 1 })]);
29 this.parts.push(b); this.at += b.length; this.n++;
30 return offset;
31 }
32 build() {
33 const h = Buffer.alloc(12); h.write('PACK'); h.writeUInt32BE(2, 4); h.writeUInt32BE(this.n, 8);
34 const body = Buffer.concat([h, ...this.parts]);
35 return new Uint8Array(Buffer.concat([body, createHash('sha1').update(body).digest()]));
36 }
37}
38// content(0) whole; a REF chain content(1..K), each on the one before by id; an OFS chain content(K+1..K+D) on the top of
39// it; M leaves content(K+D+1..K+D+M), each an OFS delta on the end of the OFS chain. Adverse: the REF chain is emitted
40// top first and the whole base after it, so every REF delta precedes its base and every OFS delta waits on all of them.
41function shape(K, D, M, adverse) {
42 const p = new Pack();
43 const refAt = [];
44 const ref = i => { refAt[i] = p.add(REF, replaceDelta(content(i - 1).length, content(i)), { ref: sha1('blob', content(i - 1)) }); };
45 if (adverse) { for (let i = K; i >= 1; i--) ref(i); p.add(BLOB, content(0)); } else { p.add(BLOB, content(0)); for (let i = 1; i <= K; i++) ref(i); }
46 let prev = refAt[K];
47 for (let i = K + 1; i <= K + D; i++) prev = p.add(OFS, replaceDelta(content(i - 1).length, content(i)), { ofs: prev });
48 for (let m = 1; m <= M; m++) p.add(OFS, replaceDelta(content(K + D).length, content(K + D + m)), { ofs: prev });
49 return { pack: p.build(), count: 1 + K + D + M };
50}
51const resolved = (pack, limits) => { const parsed = parsePack(pack, { limits }); assert.ok(parsed.ok, JSON.stringify(parsed)); return resolvePack(parsed.value, undefined, limits); };
53for (const adverse of [true, false]) {
54 test(`resolvePack's work is linear in the pack's entries, REF chain ${adverse ? 'child-first (adverse)' : 'base-first'}`, () => {
55 const limits = withLimits({ maxDeltaChain: 200 });
56 // every dimension doubled: the REF chain, the OFS chain on it, the leaves
57 const small = shape(30, 30, 500, adverse);
58 const large = shape(60, 60, 1000, adverse);
59 const a = resolved(small.pack, limits);
60 const b = resolved(large.pack, limits);
61 assert.ok(a.ok && b.ok);
62 assert.equal(a.value.objects.size, small.count);
63 assert.equal(b.value.objects.size, large.count);
64 // by hand: a REF_DELTA's base is looked up by id once, and the pack has K of them
65 assert.equal(a.value.stats.baseLookups, 30);
66 assert.equal(b.value.stats.baseLookups, 60);
67 // by hand: an entry is visited once in pack order and at most once more when the base it waited on lands
68 assert.ok(a.value.stats.steps >= small.count && a.value.stats.steps <= 2 * small.count, `steps ${a.value.stats.steps} for ${small.count} entries`);
69 assert.ok(b.value.stats.steps >= large.count && b.value.stats.steps <= 2 * large.count, `steps ${b.value.stats.steps} for ${large.count} entries`);
70 // twice the entries (1.97x), at most 2.5x the work
71 const ratio = b.value.stats.steps / a.value.stats.steps;
72 assert.ok(ratio <= 2.5, `steps grew ${ratio}x for ${large.count / small.count}x the entries`);
73 });
74}
76test('resolvePack agrees with git index-pack and cat-file on every id and every content, both orders', () => {
77 const { root, cleanup } = tmpRoot();
78 try {
79 for (const adverse of [true, false]) oracle(root, adverse);
80 } finally {
81 cleanup();
82 }
83});
84function oracle(root, adverse) {
85 {
86 const { pack, count } = shape(32, 31, 300, adverse); // the leaves are 64 deltas deep: the default maxDeltaChain exactly
87 const r = resolved(pack);
88 assert.ok(r.ok, JSON.stringify(r));
89 const repo = `${root}/oracle-${adverse ? 'adverse' : 'benign'}.git`;
90 git(root, ['init', '-q', '--bare', repo]);
91 gitBuf(repo, ['index-pack', '--stdin'], pack);
92 const out = gitBuf(repo, ['cat-file', '--batch-all-objects', '--batch']);
93 const fromGit = new Map();
94 for (let at = 0; at < out.length;) {
95 const nl = out.indexOf(10, at);
96 const [oid, type, size] = out.subarray(at, nl).toString().split(' ');
97 fromGit.set(oid, { type, data: out.subarray(nl + 1, nl + 1 + Number(size)) });
98 at = nl + 1 + Number(size) + 1;
99 }
100 assert.equal(fromGit.size, count);
101 assert.equal(r.value.objects.size, count);
102 for (const [oid, o] of fromGit) {
103 const mine = r.value.objects.get(oid);
104 assert.ok(mine, `${oid} is in git's index and not in resolvePack's result`);
105 assert.equal(o.type, 'blob');
106 assert.equal(mine.type, BLOB);
107 assert.ok(o.data.equals(mine.data), `${oid} differs`);
108 }
109 assert.equal(r.value.stats.maxChain, 64);
110 }
111}
113// The finding's shape at 100,000 links, hand-built (resolvePack takes plain objects): three REF deltas emitted child
114// first, the whole base, then an OFS chain on the top REF delta. The old resolver recursed two frames per link.
115function deepChain(D, order) {
116 const oidHex = i => sha1('blob', content(i)).toString('hex');
117 const objects = [];
118 for (const i of [3, 2, 1]) objects.push({ offset: 100 + i, type: REF, data: replaceDelta(content(i - 1).length, content(i)), base: { tag: 'ref', oid: oidHex(i - 1) } });
119 objects.push({ offset: 100, type: BLOB, data: content(0) });
120 const chain = [];
121 for (let i = 4; i <= D + 3; i++) chain.push({ offset: 100 + i, type: OFS, data: replaceDelta(content(i - 1).length, content(i)), base: { tag: 'ofs', offset: 100 + i - 1 } });
122 if (order === 'deepest-first') chain.reverse();
123 return { objects: [...objects, ...chain], last: oidHex(D + 3) };
124}
125for (const order of ['base-first', 'deepest-first']) {
126 test(`a 100,000-link OFS chain on a late REF base resolves under a raised limit and is limit_exceeded under the default: never thrown (${order})`, () => {
127 const D = 100_000;
128 const { objects, last } = deepChain(D, order);
129 // the aggregate cap charges LIMIT_OBJECT_OVERHEAD per object: 100,004 objects need more than the default 256 MiB
130 const r = resolvePack({ objects }, undefined, withLimits({ maxDeltaChain: D + 3, maxInflatedTotal: 2 ** 30 }));
131 assert.ok(r.ok, JSON.stringify(r.ok ? null : r.error));
132 assert.equal(r.value.objects.size, D + 4);
133 const deepest = r.value.objects.get(last);
134 assert.ok(deepest);
135 assert.equal(deepest.chain, D + 3);
136 assert.ok(Buffer.from(deepest.data).equals(content(D + 3)));
137 assert.equal(r.value.stats.maxChain, D + 3);
138 assert.ok(r.value.stats.steps <= 2 * (D + 4), `steps ${r.value.stats.steps}`);
139 // the default chain limit (64), by hand: the 65th delta above the whole base is the first one refused
140 const refused = resolvePack({ objects }, undefined, withLimits({ maxInflatedTotal: 2 ** 30 }));
141 assert.ok(!refused.ok);
142 assert.deepEqual([refused.error.code, refused.error.what, refused.error.got, refused.error.max], ['limit_exceeded', 'delta chain depth', 65, 64]);
143 });
144}
146test('describePack reports a 20,000-link chain without recursing, its base in the pack or thin', () => {
147 const D = 20_000;
148 const limits = withLimits({ maxDeltaChain: D + 3 });
149 const inPack = shape(3, D, 0, true);
150 const full = describePack(inPack.pack, limits);
151 assert.ok(full.ok, JSON.stringify(full.ok ? null : full.error));
152 assert.equal(full.value.objects.length, D + 4);
153 assert.equal(full.value.objects.at(-1).depth, D + 3);
154 assert.equal(full.value.objects.at(-1).oid, sha1('blob', content(D + 3)).toString('hex'));
155 // thin: one REF delta on a base that is not in the pack, the OFS chain on it. Depth as verify-pack prints it once the
156 // base is appended: the REF delta 1, each OFS link one more.
157 const p = new Pack();
158 let prev = p.add(REF, replaceDelta(content(0).length, content(1)), { ref: sha1('blob', content(0)) });
159 for (let i = 2; i <= D + 1; i++) prev = p.add(OFS, replaceDelta(content(i - 1).length, content(i)), { ofs: prev });
160 const thin = describePack(p.build(), limits);
161 assert.ok(thin.ok, JSON.stringify(thin.ok ? null : thin.error));
162 assert.deepEqual(thin.value.missingBases, [sha1('blob', content(0)).toString('hex')]);
163 assert.equal(thin.value.objects.at(-1).depth, D + 1);
164 assert.equal(thin.value.objects.at(-1).oid, null);
165 // under the default limit the same thin pack is refused with the coded error
166 const refused = describePack(p.build());
167 assert.ok(!refused.ok);
168 assert.equal(refused.error.code, 'limit_exceeded');
169});
171// ---- the leftover-wait chain bound, at its boundary (wire packs through parsePack) ----
172// A thin pack: one REF_DELTA on content(0), which is not in the pack, and `ofsCount` OFS deltas in a chain on it. Depth
173// is counted in deltas above the base (the REF_DELTA 1, the k-th OFS delta k + 1), as `git verify-pack -v` prints it.
174function thinChain(ofsCount) {
175 const p = new Pack();
176 let prev = p.add(REF, replaceDelta(content(0).length, content(1)), { ref: sha1('blob', content(0)) });
177 for (let i = 2; i <= ofsCount + 1; i++) prev = p.add(OFS, replaceDelta(content(i - 1).length, content(i)), { ofs: prev });
178 return p.build();
179}
180const counting = lookup => { const asked = []; return { asked, external: oid => { asked.push(oid); return lookup(oid); } }; };
181const coded = r => [r.error.code, r.error.what, r.error.got, r.error.max];
182test('a thin REF root with maxDeltaChain OFS deltas on it is a missing base; one more OFS delta is past the chain limit before the base is asked for', () => {
183 const CHAIN = 64; // DEFAULT_LIMITS.maxDeltaChain
184 const base = sha1('blob', content(0)).toString('hex');
185 // Expected values are not read off the new resolver: they are what main at ec337c5 (the recursive resolver) returns
186 // for these same wire packs, and what the limit means by hand. Root + 64 OFS: the deepest delta is 65 above the base,
187 // one past the limit, so it can only be refused once the base is there to patch on; with the base withheld the pack's
188 // defect is the missing base (pack_corrupt 'not found'; describePack: ok, one missing base, depth 65 as verify-pack
189 // would print it). Root + 65 OFS: the deepest is 66 above the base, and its own OFS base (65 above) is already past
190 // the limit whatever the root turns out to be, so it is limit_exceeded 65/64 and the lookup is never called.
191 {
192 const pack = thinChain(CHAIN);
193 const parsed = parsePack(pack);
194 assert.ok(parsed.ok);
195 const withheld = counting(() => null);
196 const r = resolvePack(parsed.value, withheld.external);
197 assert.ok(!r.ok);
198 assert.equal(r.error.code, 'pack_corrupt');
199 assert.equal(r.error.message, `pack: ref delta base ${base} not found`);
200 assert.deepEqual(withheld.asked, [base]);
201 const d = describePack(pack);
202 assert.ok(d.ok, JSON.stringify(d.ok ? null : d.error));
203 assert.deepEqual(d.value.missingBases, [base]);
204 assert.equal(d.value.objects.length, CHAIN + 1);
205 assert.equal(d.value.objects.at(-1).depth, CHAIN + 1);
206 // the base given: the 65th delta is refused by the patch
207 const given = counting(oid => (oid === base ? { type: BLOB, data: new Uint8Array(content(0)) } : null));
208 const g = resolvePack(parsed.value, given.external);
209 assert.ok(!g.ok);
210 assert.deepEqual(coded(g), ['limit_exceeded', 'delta chain depth', CHAIN + 1, CHAIN]);
211 assert.deepEqual(given.asked, [base]);
212 }
213 {
214 const pack = thinChain(CHAIN + 1);
215 const parsed = parsePack(pack);
216 assert.ok(parsed.ok);
217 for (const lookup of [() => null, oid => (oid === base ? { type: BLOB, data: new Uint8Array(content(0)) } : null)]) {
218 const c = counting(lookup);
219 const r = resolvePack(parsed.value, c.external);
220 assert.ok(!r.ok);
221 assert.deepEqual(coded(r), ['limit_exceeded', 'delta chain depth', CHAIN + 1, CHAIN]);
222 assert.deepEqual(c.asked, []);
223 }
224 const d = describePack(pack);
225 assert.ok(!d.ok);
226 assert.deepEqual(coded(d), ['limit_exceeded', 'delta chain depth', CHAIN + 1, CHAIN]);
227 }
228});
230// ---- hand-built input parsePack cannot produce ----
231const blobId = i => sha1('blob', content(i)).toString('hex');
232const ofsAt = (offset, baseOffset, i) => ({ offset, type: OFS, data: replaceDelta(content(i - 1).length, content(i)), base: { tag: 'ofs', offset: baseOffset } });
233const refAt = (offset, baseI, i) => ({ offset, type: REF, data: replaceDelta(content(baseI).length, content(i)), base: { tag: 'ref', oid: blobId(baseI) } });
234const blobBase = i => ({ type: BLOB, data: new Uint8Array(content(i)) });
236test('a cycle of OFS bases (hand-built) is the coded chain-depth error from resolvePack, never a throw', () => {
237 // By hand: a cycle of bases is a chain with no end, so it is past maxDeltaChain (64) at its 65th link, and a Result
238 // API reports that as a value. parsePack cannot produce one (an OFS base is strictly earlier in the pack).
239 const cycles = {
240 'self-loop': [ofsAt(10, 10, 1)],
241 '2-cycle': [ofsAt(10, 11, 1), ofsAt(11, 10, 2)],
242 '3-cycle with a tail': [ofsAt(10, 12, 1), ofsAt(11, 10, 2), ofsAt(12, 11, 3), ofsAt(13, 10, 4), ofsAt(14, 13, 5)],
243 'a tail first, then the cycle it rests on': [ofsAt(14, 13, 5), ofsAt(13, 10, 4), ofsAt(10, 11, 1), ofsAt(11, 10, 2)],
244 'a cycle beside a whole object and its delta': [{ offset: 1, type: BLOB, data: content(0) }, ofsAt(2, 1, 1), ofsAt(10, 11, 1), ofsAt(11, 10, 2)],
245 };
246 for (const [name, objects] of Object.entries(cycles)) {
247 const c = counting(() => null);
248 let r;
249 assert.doesNotThrow(() => { r = resolvePack({ objects }, c.external); }, name);
250 assert.ok(!r.ok, name);
251 assert.deepEqual(coded(r), ['limit_exceeded', 'delta chain depth', 65, 64], name);
252 assert.deepEqual(c.asked, [], name);
253 }
254});
256test('two entries hand-built at one offset are one object: the second thin REF delta at a resolved offset is not patched or counted', () => {
257 // An offset names one object, so the first entry at it is that object (the rule the resolver states for every other
258 // phase, and main at ec337c5's result for this input: 1 object, externalBases 1). By hand: one external base used once.
259 const c = counting(oid => (oid === blobId(9) ? blobBase(9) : null));
260 const r = resolvePack({ objects: [refAt(2, 9, 1), refAt(2, 9, 2)] }, c.external);
261 assert.ok(r.ok, JSON.stringify(r.ok ? null : r.error));
262 assert.deepEqual([...r.value.objects.keys()], [blobId(1)]);
263 assert.equal(r.value.stats.externalBases, 1);
264 assert.equal(r.value.stats.maxChain, 1);
265 assert.deepEqual(c.asked, [blobId(9)]);
266 // the same rule where the base is in the pack: a second whole object, a second delta already resolved, a second
267 // delta that waited (filed under a base that lands later)
268 const twoFull = resolvePack({ objects: [{ offset: 1, type: BLOB, data: content(0) }, { offset: 1, type: BLOB, data: content(1) }] });
269 assert.ok(twoFull.ok);
270 assert.deepEqual([...twoFull.value.objects.keys()], [blobId(0)]);
271 const twoDeltas = resolvePack({ objects: [{ offset: 1, type: BLOB, data: content(0) }, ofsAt(2, 1, 1), { ...ofsAt(2, 1, 1), data: replaceDelta(content(0).length, content(2)) }] });
272 assert.ok(twoDeltas.ok);
273 assert.deepEqual([...twoDeltas.value.objects.keys()], [blobId(0), blobId(1)]);
274 const twoWaiting = resolvePack({ objects: [refAt(2, 0, 1), refAt(2, 0, 2), { offset: 1, type: BLOB, data: content(0) }] });
275 assert.ok(twoWaiting.ok);
276 assert.deepEqual([...twoWaiting.value.objects.keys()], [blobId(0), blobId(1)]);
277 // two deltas at offset 3 filed under the delta at offset 2, which resolves after them: the first is the object
278 const twoFiled = resolvePack({ objects: [ofsAt(3, 2, 2), { ...ofsAt(3, 2, 2), data: replaceDelta(content(1).length, content(3)) }, refAt(2, 0, 1), { offset: 1, type: BLOB, data: content(0) }] });
279 assert.ok(twoFiled.ok);
280 assert.deepEqual([...twoFiled.value.objects.keys()], [blobId(0), blobId(1), blobId(2)]);
281});
283test('`external` is asked once per distinct missing base id, in pack order, however many deltas wait on it', () => {
284 // By hand: 3 deltas wait on content(7), 2 on content(8), 1 on content(9): three ids, so three calls, in the order
285 // their first REF_DELTA appears in the pack, whether the lookup has the base or not.
286 const objects = [refAt(1, 7, 1), refAt(2, 8, 2), refAt(3, 7, 3), refAt(4, 9, 4), refAt(5, 7, 5), refAt(6, 8, 6)];
287 const order = [blobId(7), blobId(8), blobId(9)];
288 const have = counting(oid => (order.includes(oid) ? blobBase(order.indexOf(oid) + 7) : null));
289 const r = resolvePack({ objects }, have.external);
290 assert.ok(r.ok, JSON.stringify(r.ok ? null : r.error));
291 assert.equal(r.value.objects.size, 6);
292 assert.equal(r.value.stats.externalBases, 6);
293 assert.deepEqual(have.asked, order);
294 const none = counting(() => null);
295 const refused = resolvePack({ objects }, none.external);
296 assert.ok(!refused.ok);
297 assert.equal(refused.error.message, `pack: ref delta base ${blobId(7)} not found`);
298 assert.deepEqual(none.asked, order);
299 // one id has it, the others do not: still one call each
300 const some = counting(oid => (oid === blobId(8) ? blobBase(8) : null));
301 const partly = resolvePack({ objects }, some.external);
302 assert.ok(!partly.ok);
303 assert.deepEqual(some.asked, order);
304});
306test('a thin REF delta whose bytes do not apply to the base `external` gives is the patch error, not a missing base and not ok', () => {
307 // By the pack format a delta opens with the size of the base it was made against; this one says 12 for an 11-byte
308 // base, so no object can be built from it. The base was found, so the defect is the delta: main at ec337c5 returns
309 // pack_corrupt 'delta: base size 12 != 11' for each of these inputs (not `ref delta base … not found`). The base is
310 // asked for once: one id (the rule pinned above; main asked once per waiting delta).
311 const BAD_DELTA = 'delta: base size 12 != 11';
312 const bad ={ offset: 2, type: REF, data: replaceDelta(content(9).length + 1, content(1)), base: { tag: 'ref', oid: blobId(9) } };
313 for (const objects of [[bad], [bad, ofsAt(3, 2, 2)], [refAt(1, 9, 3), bad]]) {
314 const c = counting(oid => (oid === blobId(9) ? blobBase(9) : null));
315 let r;
316 assert.doesNotThrow(() => { r = resolvePack({ objects }, c.external); });
317 assert.ok(!r.ok);
318 assert.equal(r.error.code, 'pack_corrupt');
319 assert.equal(r.error.message, BAD_DELTA);
320 assert.deepEqual(c.asked, [blobId(9)]);
321 }
322});
324test('a REF delta whose base is built from an external base is in the pack after all: `external` is not asked for it', () => {
325 // By hand: content(1) is made in this pack (a REF delta on content(9), which the caller holds), so the second delta's
326 // base is not missing and only content(9) is asked for. Main at ec337c5 asks [content(9)] too.
327 for (const objects of [[refAt(2, 9, 1), refAt(3, 1, 2)], [refAt(2, 9, 1), refAt(3, 1, 2), refAt(4, 2, 3), ofsAt(5, 4, 4)]]) {
328 const c = counting(oid => (oid === blobId(9) ? blobBase(9) : null));
329 const r = resolvePack({ objects }, c.external);
330 assert.ok(r.ok, JSON.stringify(r.ok ? null : r.error));
331 assert.deepEqual(c.asked, [blobId(9)]);
332 assert.equal(r.value.objects.size, objects.length);
333 assert.equal(r.value.stats.externalBases, 1);
334 }
335});
337test('stats.steps on thin packs is the count the visit rule gives: three per delta that waits on an external base, one per object that does not wait', () => {
338 // The rule (README, stats.steps): a whole object is visited once; a delta whose base has resolved when it is reached
339 // is visited once (patched); a delta that waits is visited three times: filed, walked once to the id it waits on,
340 // patched once the base is given. The expected numbers below are that rule applied to each pack's shape by hand.
341 {
342 // one thin REF root and 2,000 OFS deltas in a chain above it: 2,001 deltas, all waiting on the one external base
343 const K = 2000;
344 const objects = [refAt(100, 0, 1)];
345 for (let i = 2; i <= K + 1; i++) objects.push(ofsAt(99 + i, 98 + i, i));
346 const c = counting(oid => (oid === blobId(0) ? blobBase(0) : null));
347 const r = resolvePack({ objects }, c.external, withLimits({ maxDeltaChain: 1e6 }));
348 assert.ok(r.ok, JSON.stringify(r.ok ? null : r.error));
349 assert.equal(r.value.objects.size, K + 1);
350 assert.equal(r.value.stats.steps, 3 * (K + 1));
351 assert.equal(r.value.stats.maxChain, K + 1);
352 assert.deepEqual(c.asked, [blobId(0)]);
353 }
354 {
355 // 10 whole blobs, each with a 5-delta OFS chain after it (60 objects that never wait: 60 steps), then 3 thin REF
356 // roots, each with 4 OFS deltas above it (15 deltas that wait: 45 steps)
357 const objects = [];
358 let at = 1, next = 100;
359 for (let g = 0; g < 10; g++) {
360 objects.push({ offset: at, type: BLOB, data: content(next++) });
361 for (let j = 0; j < 5; j++) { objects.push(ofsAt(at + 1, at, next++)); at++; }
362 at++;
363 }
364 const bases = [900, 901, 902];
365 for (const b of bases) {
366 objects.push(refAt(at, b, next++));
367 for (let j = 0; j < 4; j++) { objects.push(ofsAt(at + 1, at, next++)); at++; }
368 at++;
369 }
370 const c = counting(oid => { const i = bases.findIndex(b => blobId(b) === oid); return i < 0 ? null : blobBase(bases[i]); });
371 const r = resolvePack({ objects }, c.external);
372 assert.ok(r.ok, JSON.stringify(r.ok ? null : r.error));
373 assert.equal(r.value.objects.size, 75);
374 assert.equal(r.value.stats.steps, 60 + 3 * 15);
375 assert.equal(r.value.stats.externalBases, 3);
376 assert.equal(r.value.stats.maxChain, 5);
377 assert.deepEqual(c.asked, bases.map(blobId));
378 }
379});
381test('every REF delta filed under an id `external` gives is patched on that base, also when the pack then builds that id itself', () => {
382 // Hand-built (parsePack can produce the shape from wire bytes; git does not write it):
383 // REF@2 on c9 -> c1, OFS@3 on offset 2 -> c9, REF@4 on c9 -> c5, `external` holds c9 only.
384 // Nothing resolves from the pack alone, so c9 is asked for, once. Both REF deltas were filed under c9 before it was
385 // asked for, and the README rule is that every REF_DELTA waiting on an id `external` gives is patched directly on that
386 // base. Applied by hand: @2 is one delta above the given c9 (chain 1, external base 1); landing it lets @3 resolve on
387 // it, which builds c9 again inside the pack (chain 2); @4 is still patched on the given base (chain 1, external base
388 // 2), not on the pack's copy at @3 (which would be chain 3 and external base 1). So: 3 objects, externalBases 2,
389 // maxChain 2, c5 at chain 1, and a limit of 2 deltas is enough. Ids are node:crypto's.
390 // ofsAt(offset, baseOffset, i) writes content(i - 1).length as its delta's source size while the base here is
391 // content(1): every content(i) is 11 bytes, so the size matches and the delta applies.
392 assert.equal(content(8).length, content(1).length);
393 const objects = [refAt(2, 9, 1), ofsAt(3, 2, 9), refAt(4, 9, 5)];
394 for (const limits of [undefined, withLimits({ maxDeltaChain: 2 })]) {
395 const c = counting(oid => (oid === blobId(9) ? blobBase(9) : null));
396 const r = resolvePack({ objects }, c.external, limits);
397 assert.ok(r.ok, JSON.stringify(r.ok ? null : r.error));
398 assert.deepEqual(c.asked, [blobId(9)]);
399 assert.equal(r.value.objects.size, 3);
400 assert.deepEqual([...r.value.objects.keys()].sort(), [blobId(1), blobId(5), blobId(9)].sort());
401 assert.equal(r.value.stats.externalBases, 2);
402 assert.equal(r.value.stats.maxChain, 2);
403 assert.equal(r.value.objects.get(blobId(5)).chain, 1);
404 }
405});
407test('a REF delta whose base the pack builds is patched on the pack\'s copy: `external` is not asked for that id even if it holds it, and the chain is the pack\'s own', () => {
408 // Hand-built (parsePack can produce the shape from wire bytes; git writes a base before its deltas):
409 // REF@2 on X -> A, REF@3 on B -> C, OFS@4 on offset 2 -> B, `external` holds X and B (X = c8, A = c1, B = c2, C = c3).
410 // The README rule: `external` supplies a base that is not in the pack, and a REF_DELTA whose base the pack builds is
411 // patched on the pack's copy whenever that copy resolves before the id is asked for. Applied by hand: nothing resolves
412 // from the pack alone (@2 waits on X, @3 on B, @4 on @2), so the first waiting id in pack order, X, is asked for. @2 is
413 // patched on the given X (A: chain 1, external base 1). That lets @4 resolve on @2, which builds B inside the pack
414 // (chain 2). B is now in the pack before it was asked for, so @3 is patched on the pack's B: C is chain 2 + 1 = 3, and
415 // `external` is never asked for B though it holds it. So: asked [X], 3 objects, externalBases 1, maxChain 3, C at
416 // chain 3; and a limit of 2 deltas refuses the pack at C's third link (got 3, max 2), still having asked only X.
417 // (Main at ec337c5 asked X then B and patched C on the given B: chain 1, externalBases 2, ok at a limit of 2.)
418 // ofsAt(4, 2, 2) writes content(1).length as its source size and its base here is content(1): the delta applies.
419 const [X, A, B, C] = [8, 1, 2, 3];
420 const objects = [refAt(2, X, A), refAt(3, B, C), ofsAt(4, 2, B)];
421 const held = oid => (oid === blobId(X) ? blobBase(X) : oid === blobId(B) ? blobBase(B) : null);
422 const c = counting(held);
423 const r = resolvePack({ objects }, c.external);
424 assert.ok(r.ok, JSON.stringify(r.ok ? null : r.error));
425 assert.deepEqual(c.asked, [blobId(X)]);
426 assert.deepEqual([...r.value.objects.keys()].sort(), [blobId(A), blobId(B), blobId(C)].sort());
427 assert.equal(r.value.stats.externalBases, 1);
428 assert.equal(r.value.stats.maxChain, 3);
429 assert.equal(r.value.objects.get(blobId(A)).chain, 1);
430 assert.equal(r.value.objects.get(blobId(B)).chain, 2);
431 assert.equal(r.value.objects.get(blobId(C)).chain, 3);
432 const limited = counting(held);
433 let refused;
434 assert.doesNotThrow(() => { refused = resolvePack({ objects }, limited.external, withLimits({ maxDeltaChain: 2 })); });
435 assert.ok(!refused.ok);
436 assert.deepEqual(coded(refused), ['limit_exceeded', 'delta chain depth', 3, 2]);
437 assert.equal(refused.error.message, 'delta chain depth 3 exceeds limit 2');
438 assert.deepEqual(limited.asked, [blobId(X)]);
439});
Diagnostics6 git calls · 321 ms
CallStatusSizeTime0321 ms
discover200191 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB86 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=86 git.response.bytes=11990 git.response.cache=EXPIRED git.response.duration_ms=86 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetch2007.8 KiB227 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=227 git.response.bytes=8020 git.response.cache=MISS git.response.duration_ms=227 git.fetch.wants=238a07242cd467a6bfe2c4ba5737762803877b22 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=8020 git.pkt.payload.max=7993 git.sideband.band1.bytes=7993 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e14786e8efa140
- Request
- 7d3d3918…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format