GitHubivarvong
ply
main
1// resolvePackRange's work bound and rebuild cost, on packs built here with node:zlib and node:crypto (no ply encoder):
2// a delta bomb is refused at limits.maxResolveWork under each preset, and a forked delta family larger than the cache
3// resolves within a small multiple of its object bytes.
4import { test } from 'vitest';
5import assert from 'node:assert/strict';
6import { deflateSync } from 'node:zlib';
7import { createHash } from 'node:crypto';
8import { execFileSync } from 'node:child_process';
9import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
10import { tmpdir } from 'node:os';
11import { join } from 'node:path';
12import { TYPE } from '../src/objects.js';
13import { scanPackIndex } from '../src/packscanindex.js';
14import { resolvePackRange } from '../src/packrangeresolve.js';
15import { rangeFromBytes } from '../src/rangesource.js';
16import { DEFAULT_LIMITS, WORKER_LIMITS, withLimits, withDerivedWork, parseLimits } from '../src/limits.js';
17import { unwrap } from './unwrap.js';
18import { RETAIN_BUDGET } from '../src/packstream.js';
20const MIB = 1 << 20;
21const cat = parts => { const o = new Uint8Array(parts.reduce((a, p) => a + p.length, 0)); let at = 0; for (const p of parts) { o.set(p, at); at += p.length; } return o; };
22const oidOf = data => createHash('sha1').update(`blob ${data.length}\0`).update(data).digest('hex');
23function entryHeader(type, size) { const out = []; let b = (type << 4) | (size & 15); size = Math.floor(size / 16); while (size > 0) { out.push(b | 0x80); b = size & 0x7f; size = Math.floor(size / 128); } out.push(b); return Uint8Array.from(out); }
24function ofsEnc(n) { const out = [n & 0x7f]; n = Math.floor(n / 128); while (n > 0) { n -= 1; out.unshift(0x80 | (n & 0x7f)); n = Math.floor(n / 128); } return Uint8Array.from(out); }
25function varint(n) { const out = []; while (n >= 128) { out.push((n & 0x7f) | 0x80); n = Math.floor(n / 128); } out.push(n); return out; }
26function copyOp(off, size) { const bytes = []; let cmd = 0x80; for (let i = 0; i < 4; i++) { const v = (off >>> (8 * i)) & 0xff; if (v) { cmd |= 1 << i; bytes.push(v); } } for (let i = 0; i < 3; i++) { const v = (size >>> (8 * i)) & 0xff; if (v) { cmd |= 0x10 << i; bytes.push(v); } } return [cmd, ...bytes]; }
27// entries: { data } (a blob) or { ofs: index of the base, delta }
28function buildPack(entries) {
29 const n = entries.length;
30 const parts = [Uint8Array.from([0x50, 0x41, 0x43, 0x4b, 0, 0, 0, 2, (n >>> 24) & 255, (n >>> 16) & 255, (n >>> 8) & 255, n & 255])];
31 let at = 12; const offs = [];
32 for (const e of entries) {
33 offs.push(at);
34 const body = deflateSync(e.ofs === undefined ? e.data : e.delta, { level: 1 });
35 const head = e.ofs === undefined ? entryHeader(3, e.data.length) : cat([entryHeader(6, e.delta.length), ofsEnc(at - offs[e.ofs])]);
36 parts.push(head, body); at += head.length + body.length;
37 }
38 const all = cat(parts);
39 return cat([all, createHash('sha1').update(all).digest()]);
40}
41const streamOf = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
43// One `unit`-byte blob of zeros and `n` deltas on it, each a list of whole-blob copies declaring `result` bytes, the
44// last 4 naming the delta so every result is a distinct object.
45function bomb(n, unit, result) {
46 const entries = [{ data: new Uint8Array(unit) }];
47 for (let i = 0; i < n; i++) {
48 const ops = [];
49 for (let k = 0; k < result / unit - 1; k++) ops.push(...copyOp(0, unit));
50 ops.push(...copyOp(0, unit - 4), 4, i & 255, (i >> 8) & 255, 0x55, 0xaa);
51 entries.push({ ofs: 0, delta: Uint8Array.from([...varint(unit), ...varint(result), ...ops]) });
52 }
53 return buildPack(entries);
54}
55async function resolveBomb(pack, limits) {
56 const idx = await scanPackIndex(streamOf(pack), { limits });
57 assert.equal(idx.ok, true, idx.ok ? '' : idx.error.message);
58 return resolvePackRange(idx.value, rangeFromBytes(pack), { limits });
59}
60// The refusal: at the ceiling, having done at most one delta (its instructions and its result) more than the ceiling.
61function assertRefusedAt(r, ceiling, result) {
62 assert.equal(r.ok, false, 'the bomb resolved');
63 assert.equal(r.error.code, 'limit_exceeded');
64 assert.equal(r.error.what, 'resolve work bytes');
65 assert.equal(r.error.max, ceiling);
66 assert.ok(r.error.got <= ceiling + result + 1024, `work ${r.error.got} ran past ${ceiling} by more than one delta`);
67}
69test('a delta bomb under WORKER_LIMITS is refused at 512 MiB of work, whatever the pack declares', { timeout: 120_000 }, async () => {
70 // 40 deltas each declaring 16 MiB (WORKER_LIMITS.maxDeltaResult): 640 MiB of results from a pack of a few KB
71 const pack = bomb(40, MIB, 16 * MIB);
72 assert.ok(pack.length < 8192, `pack is ${pack.length} bytes`);
73 // by hand: 8 x maxInflatedTotal (64 MiB) = 536,870,912
74 assertRefusedAt(await resolveBomb(pack, WORKER_LIMITS), 536_870_912, 16 * MIB);
75});
77test('a delta bomb under DEFAULT_LIMITS is refused at 2 GiB of work', { timeout: 300_000 }, async () => {
78 // 40 deltas each declaring 64 MiB (DEFAULT_LIMITS.maxDeltaResult): 2.5 GiB of results
79 const pack = bomb(40, MIB, 64 * MIB);
80 // by hand: 8 x maxInflatedTotal (256 MiB) = 2,147,483,648
81 assertRefusedAt(await resolveBomb(pack, DEFAULT_LIMITS), 2_147_483_648, 64 * MIB);
82});
84test('the work ceiling follows the limits a caller gives: left out, it is 8 x their maxInflatedTotal', async () => {
85 const pack = bomb(40, 64 * 1024, MIB);
86 // by hand: 8 x 4 MiB = 33,554,432, for a partial record and for withLimits alike
87 const partial = { maxInflatedTotal: 4 * MIB, maxDeltaResult: MIB, maxObjectSize: MIB };
88 assertRefusedAt(await resolveBomb(pack, partial), 33_554_432, MIB);
89 assertRefusedAt(await resolveBomb(pack, withLimits(partial)), 33_554_432, MIB);
90 // given, it is what was given: 40 MiB of results pass under 64 MiB and are refused under 6 MiB
91 const roomy = await resolveBomb(pack, { ...partial, maxResolveWork: 64 * MIB });
92 assert.equal(roomy.ok, true);
93 assert.equal(roomy.value.stats.workMax, 64 * MIB);
94 assertRefusedAt(await resolveBomb(pack, { ...partial, maxResolveWork: 6 * MIB }), 6 * MIB, MIB);
95 // #271: the ceiling is the parsed maxResolveWork, never raised. At the record's own maxInflatedTotal (4 MiB, the least
96 // it may be) the bomb is refused at 4,194,304; given below it (2 MiB) the record is bad_request before any read
97 for (const work of [4 * MIB, 6 * MIB, 64 * MIB]) {
98 const record = { ...partial, maxResolveWork: work };
99 const r = await resolveBomb(pack, record);
100 assert.equal(r.ok ? r.value.stats.workMax : r.error.max, unwrap(parseLimits(record)).maxResolveWork, `${work}`);
101 }
102 assertRefusedAt(await resolveBomb(pack, { ...partial, maxResolveWork: 4 * MIB }), 4_194_304, MIB);
103 const idx = await scanPackIndex(streamOf(pack), { limits: partial });
104 const under = await resolvePackRange(idx.value, rangeFromBytes(pack), { limits: { ...partial, maxResolveWork: 2 * MIB } });
105 assert.equal(under.ok, false);
106 assert.equal(under.error.code, 'bad_request');
107 assert.equal(under.error.what, 'maxResolveWork');
108 assert.equal(DEFAULT_LIMITS.maxResolveWork, 2_147_483_648);
109 assert.equal(WORKER_LIMITS.maxResolveWork, 536_870_912);
110});
112test('maxResolveWork: null is bad_request like every key (#271); derived is the key left out, or withDerivedWork on a spread preset', async () => {
113 // by hand: 8 x 4 MiB = 33,554,432, whatever the record is spread from
114 const partial = { maxInflatedTotal: 4 * MIB, maxDeltaResult: MIB, maxObjectSize: MIB };
115 for (const record of [{ ...partial, maxResolveWork: null }, { ...WORKER_LIMITS, ...partial, maxResolveWork: null }, { ...DEFAULT_LIMITS, ...partial, maxResolveWork: null }]) {
116 const parsed = parseLimits(record);
117 assert.equal(parsed.ok, false);
118 assert.equal(parsed.error.code, 'bad_request');
119 assert.equal(parsed.error.what, 'maxResolveWork');
120 assert.equal(parsed.error.got, null);
121 }
122 for (const record of [partial, withDerivedWork({ ...WORKER_LIMITS, ...partial }), withDerivedWork({ ...DEFAULT_LIMITS, ...partial })]) {
123 assert.equal(Object.hasOwn(record, 'maxResolveWork'), false);
124 assert.equal(unwrap(parseLimits(record)).maxResolveWork, 33_554_432);
125 assert.equal(withLimits(record).maxResolveWork, 33_554_432);
126 }
127 assertRefusedAt(await resolveBomb(bomb(40, 64 * 1024, MIB), withDerivedWork({ ...WORKER_LIMITS, ...partial })), 33_554_432, MIB);
128});
130test('withLimits(null) is the frozen defaults: null from an untyped caller is no overrides, it does not throw', () => {
131 // every value written out from the documented defaults (README Limits; MiB = 1,048,576), none read from DEFAULT_LIMITS
132 const defaults = {
133 maxMergeBlobBytes: 8_388_608, maxObjectSize: 67_108_864, maxHarvestBytes: 33_554_432, maxPackBytes: 134_217_728, maxPushBytes: 134_217_728,
134 maxDeltaResult: 67_108_864, maxDeltaChain: 64, maxInflatedTotal: 268_435_456, maxResolveWork: 2_147_483_648, maxPackObjects: 100_000,
135 maxRefs: 10_000, maxRebaseCommits: 10_000, maxSectionLines: 1000, maxTreeEntries: 200_000, maxPathDepth: 128, maxPathComponent: 4096,
136 maxMergeDepth: 1024, maxReportLines: 10_000, maxCommitParents: 10_000, maxLsRefsBytes: 134_217_728, requestTimeoutMs: 0,
137 firstByteTimeoutMs: 60_000, idleTimeoutMs: 60_000,
138 // README Limits: heldTimeoutMs 240 s, 4 x the 60 s idle default (#273)
139 heldTimeoutMs: 240_000,
140 // README Limits: maxMergeWork 2^32 diff steps, maxReportReasonBytes 4096, maxReportProgressBytes 8 MiB
141 maxMergeWork: 4_294_967_296, maxReportReasonBytes: 4096, maxReportProgressBytes: 8_388_608,
142 };
143 const made = withLimits(null);
144 assert.deepEqual({ ...made }, defaults);
145 assert.equal(Object.isFrozen(made), true);
146 const parsed = parseLimits(withLimits(null));
147 assert.equal(parsed.ok, true);
148 assert.deepEqual({ ...parsed.value }, defaults);
149});
151test('maxResolveWork is a limits key like any other: NaN, Infinity or negative is bad_request under either budget, and the refusal names limits.maxResolveWork', async () => {
152 const pack = bomb(2, 1024, 1024);
153 const idx = await scanPackIndex(streamOf(pack), {});
154 for (const bad of [Number.NaN, Number.POSITIVE_INFINITY, -1, '100']) {
155 for (const budget of [RETAIN_BUDGET, { tag: 'work', ratio: 32, capObjects: false }]) {
156 const r = await resolvePackRange(idx.value, rangeFromBytes(pack), { limits: { maxResolveWork: bad }, budget });
157 assert.equal(r.ok, false, `maxResolveWork ${bad} was accepted under ${budget.tag}`);
158 assert.equal(r.error.code, 'bad_request');
159 assert.equal(r.error.message, 'limits.maxResolveWork must be a finite non-negative number');
160 assert.equal(r.error.what, 'maxResolveWork');
161 }
162 assert.equal(parseLimits({ maxResolveWork: bad }).ok, false);
163 }
164});
166test('a forked family larger than the cache: a sibling that missed the cache costs one pass, not its whole chain again', async () => {
167 // A spine 50 deep; at each level a second delta on the same base (written first) with one leaf on it: 151 distinct
168 // objects of S bytes, depth 51, every one zeros but its last 4 bytes. The cache holds 16 of them.
169 const S = 64 * 1024; const D = 50;
170 const entries = [{ data: new Uint8Array(S) }];
171 const want = [oidOf(new Uint8Array(S))];
172 let tag = 1;
173 const edit = () => {
174 const t = tag++;
175 const obj = new Uint8Array(S); obj.set([t & 255, (t >> 8) & 255, 0x55, 0xaa], S - 4);
176 want.push(oidOf(obj));
177 return Uint8Array.from([...varint(S), ...varint(S), ...copyOp(0, S - 4), 4, t & 255, (t >> 8) & 255, 0x55, 0xaa]);
178 };
179 let spine = 0;
180 for (let k = 1; k <= D; k++) {
181 entries.push({ ofs: spine, delta: edit() }); const e = entries.length - 1;
182 entries.push({ ofs: spine, delta: edit() }); const s = entries.length - 1;
183 entries.push({ ofs: e, delta: edit() });
184 spine = s;
185 }
186 const pack = buildPack(entries);
187 const idx = await scanPackIndex(streamOf(pack), {});
188 assert.equal(idx.ok, true);
189 const inner = rangeFromBytes(pack);
190 let reads = 0;
191 const source = { size: inner.size, read: (o, l) => { reads += 1; return inner.read(o, l); } };
192 const r = await resolvePackRange(idx.value, source, { cacheBytes: 16 * S });
193 assert.equal(r.ok, true, r.ok ? '' : r.error.message);
194 // every oid, from node:crypto over the bytes the edits describe
195 for (let i = 0; i < want.length; i++) assert.equal(r.value.oid(i).value, want[i], `oid of entry ${i}`);
196 // by hand: 151 entries, 151 x S object bytes. Each object must be read and made once; twice that is the bound.
197 assert.equal(want.length, 151);
198 assert.ok(reads <= 2 * 151, `${reads} range reads for 151 entries`);
199 assert.ok(r.value.stats.work <= 2 * 151 * S, `${r.value.stats.work} work bytes for ${151 * S} bytes of objects`);
200});
202test('a spread of WORKER_LIMITS with maxInflatedTotal raised to 1 GiB resolves a real git history of more than 512 MiB: the work bound is never below the record\'s maxInflatedTotal', { timeout: 600_000 }, async () => {
203 const dir = mkdtempSync(join(tmpdir(), 'ply-range-spread-'));
204 try {
205 const git = (args, input) => execFileSync('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.com', '-c', 'core.compression=1', ...args], { cwd: dir, input, maxBuffer: 1 << 30 });
206 git(['init', '-q', '.']);
207 // a lockfile-like text file of about 13 MB (under WORKER_LIMITS.maxObjectSize, 16 MiB); each commit changes one line
208 const lines = Array.from({ length: 108_000 }, (_, i) => ` "pkg-${i}": { "version": "1.0.${i}", "integrity": "sha512-${((i * 2654435761) >>> 0).toString(16).padStart(8, '0').repeat(8)}" },`);
209 for (let c = 0; c < 48; c++) {
210 lines[(c * 449) % lines.length] = ` "edited-${c}": { "version": "2.0.${c}" },`;
211 writeFileSync(join(dir, 'package-lock.json'), lines.join('\n'));
212 git(['add', 'package-lock.json']);
213 git(['commit', '-q', '-m', `edit ${c}`]);
214 }
215 const pack = new Uint8Array(git(['pack-objects', '--all', '--stdout', '-q', '--delta-base-offset'], ''));
216 // git's own list of the repository's objects: oid, type, size
217 const want = git(['cat-file', '--batch-all-objects', '--batch-check']).toString().trim().split('\n');
218 const sum = want.reduce((a, l) => a + Number(l.split(' ')[2]), 0);
219 // the case needs objects summing past the preset's maxResolveWork (512 MiB) and under the raised maxInflatedTotal (1 GiB)
220 assert.ok(sum > 536_870_912 && sum < 1_073_741_824, `history ${sum} bytes in a pack of ${pack.length}`);
221 // the preset with maxInflatedTotal raised past its maxResolveWork (512 MiB) is bad_request (#271): raise both
222 assert.equal(parseLimits({ ...WORKER_LIMITS, maxInflatedTotal: 1024 * MIB }).error.what, 'maxResolveWork');
223 const limits = { ...WORKER_LIMITS, maxInflatedTotal: 1024 * MIB, maxResolveWork: 1024 * MIB };
224 const idx = await scanPackIndex(streamOf(pack), { limits });
225 assert.equal(idx.ok, true, idx.ok ? '' : idx.error.message);
226 const r = await resolvePackRange(idx.value, rangeFromBytes(pack), { limits });
227 assert.equal(r.ok, true, r.ok ? '' : `${r.error.code}: ${r.error.message}`);
228 const nameOf = { [TYPE.COMMIT]: 'commit', [TYPE.TREE]: 'tree', [TYPE.BLOB]: 'blob', [TYPE.TAG]: 'tag' };
229 const got = Array.from({ length: r.value.count }, (_, i) => `${r.value.oid(i).value} ${nameOf[r.value.type(i).value]} ${r.value.columns.size[i]}`);
230 assert.deepEqual(got.toSorted(), want.toSorted());
231 // by hand: the maxResolveWork given, 1024 x 1,048,576 = 1,073,741,824
232 assert.equal(r.value.stats.workMax, 1_073_741_824);
233 assert.ok(r.value.stats.work > 536_870_912, `work ${r.value.stats.work} did not pass the preset's 512 MiB`);
234 console.log(`spread history: pack ${pack.length} B, ${r.value.count} objects, sum of sizes ${sum}, work ${r.value.stats.work}`);
235 } finally {
236 rmSync(dir, { recursive: true, force: true });
237 }
238});
Diagnostics6 git calls · 371 ms
CallStatusSizeTime0371 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB96 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=94 git.response.bytes=11990 git.response.cache=EXPIRED git.response.duration_ms=94 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetch2005.2 KiB267 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=267 git.response.bytes=5351 git.response.cache=MISS git.response.duration_ms=267 git.fetch.wants=46929418cdf9e9aadef2e7b37ed84671bcd9930c git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5351 git.pkt.payload.max=5324 git.sideband.band1.bytes=5324 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2a996a846e6b0
- Request
- 715a9805…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format