GitHubivarvong

ply

main
ply6 git calls48 ms26 KiB5 cachedprotocol v2
180 lines8.2 KiBRaw
1// Issue #140. listTree of a tree at maxTreeEntries used to die in a 128 MiB heap, and the refusal one
2// entry past the cap ran only after the array of entries already existed (~350 MB RSS). The count of
3// framed entries happens first; the hex id is a getter. Git is the oracle for a tree at the cap (git
4// stores it). One entry past the cap is ply's typed limit, which git does not have.
5import { test } from 'vitest';
6import assert from 'node:assert/strict';
7import { createHash } from 'node:crypto';
8import { spawn } from 'node:child_process';
9import { mkdtempSync, readFileSync, writeFileSync, rmSync } from 'node:fs';
10import { tmpdir } from 'node:os';
11import { join } from 'node:path';
12import { parseTree, encodeTree } from '../src/objects.js';
13import { toHex } from '../src/sha1.js';
14import { LIMIT_TREE_ENTRIES } from '../src/limits.js';
15import { OID_BYTES, TREE_ENTRY_MIN } from '../src/format.js';
16import { te } from '../src/bytes.js';
17import { git, gitBuf, tmpRoot } from './server.js';
18import { unwrap } from './unwrap.js';
20const MODE = '100644 ';
21const NAME = 8;
22const STRIDE = MODE.length + NAME + 1 + OID_BYTES;
23const CAP = LIMIT_TREE_ENTRIES;
24// Measured on this tree (zero-padded names, listTree-shaped map): about 242 MB at the cap and 125 MB
25// on the refusal. The old refusal was ~270–350 MB and the cap OOM'd a 128 MiB heap. Headroom for CI.
26const RSS_BOUND_MB = 320;
28const entryAt = (buf, i) => {
29 const o = i * STRIDE;
30 buf.write(MODE, o, 'ascii');
31 buf.write(String(i).padStart(NAME, '0'), o + MODE.length, 'ascii');
32 buf[o + MODE.length + NAME] = 0;
33 const id = o + MODE.length + NAME + 1;
34 buf[id] = 1;
35 buf[id + OID_BYTES - 1] = i & 0xff;
36 buf[id + OID_BYTES - 2] = (i >>> 8) & 0xff;
37 buf[id + OID_BYTES - 3] = (i >>> 16) & 0xff;
38};
40const build = n => {
41 const buf = Buffer.alloc(n * STRIDE);
42 for (let i = 0; i < n; i++) entryAt(buf, i);
43 return buf;
44};
46const runChild = (file, count) => new Promise((resolve, reject) => {
47 const child = spawn(process.execPath, [
48 '--max-old-space-size=128', '--import', './test/ts-resolve.mjs',
49 'test/tree-budget-child.mjs', file, String(STRIDE), String(count),
50 ], { stdio: ['ignore', 'pipe', 'pipe'] });
51 const out = [];
52 let err = '';
53 child.stdout.on('data', d => out.push(d));
54 child.stderr.on('data', d => { err += d; });
55 child.on('close', status => {
56 const text = Buffer.concat(out).toString();
57 let parsed = null;
58 try { parsed = JSON.parse(text); } catch { /* the heap killer prints no JSON */ }
59 resolve({ status, parsed, err, text });
60 });
61 child.on('error', reject);
62});
64const throws = (fn, code, message) => {
65 assert.throws(fn, e => e.code === code && e.message === message);
66};
68test('a framed entry past the cap is limit_exceeded, and a bad prefix is still pack_corrupt', () => {
69 const oid = Uint8Array.of(1, ...new Uint8Array(OID_BYTES - 1));
70 const raw = (text, id = oid) => {
71 const body = te.encode(text);
72 const out = new Uint8Array(body.length + id.length);
73 out.set(body);
74 out.set(id, body.length);
75 return out;
76 };
77 const data = (...parts) => {
78 const n = parts.reduce((s, p) => s + p.length, 0);
79 const out = new Uint8Array(n);
80 let p = 0;
81 for (const part of parts) { out.set(part, p); p += part.length; }
82 return out;
83 };
84 throws(() => unwrap(parseTree(data(raw('100644 a\0'), raw('100644 b\0')), { maxTreeEntries: 1 })), 'limit_exceeded', 'tree entries 2 exceeds limit 1');
85 // Bytes after the cap are not parsed: a broken mode there does not become pack_corrupt.
86 throws(() => unwrap(parseTree(data(raw('100644 a\0'), te.encode('nope')), { maxTreeEntries: 1 })), 'limit_exceeded', 'tree entries 2 exceeds limit 1');
87 throws(() => unwrap(parseTree(raw('100644 .\0'), { maxTreeEntries: 1 })), 'pack_corrupt', 'tree: bad name');
88 // The duplicate sits inside the cap, so it is reported even though another entry follows.
89 throws(
90 () => unwrap(parseTree(data(raw('100644 a\0'), raw('100644 a\0', Uint8Array.of(2, ...new Uint8Array(OID_BYTES - 1))), raw('100644 b\0')), { maxTreeEntries: 2 })),
91 'pack_corrupt',
92 'tree: duplicate entry "a"',
93 );
94 const parsed = unwrap(parseTree(raw('100644 a\0')));
95 assert.equal(parsed.length, 1);
96 assert.equal(Object.hasOwn(parsed[0], 'oid'), false);
97 assert.deepEqual(Object.keys(parsed[0]).sort(), ['entryOffset', 'mode', 'name', 'oidOffset']);
98 assert.equal(parsed[0].oid, toHex(oid));
99 assert.ok(unwrap(encodeTree(parsed)).length >= TREE_ENTRY_MIN);
100});
102test('199999 and 200000 list under a 128 MiB heap; 200001 is a typed refusal and git agrees at the cap', async () => {
103 const dir = mkdtempSync(join(tmpdir(), 'ply-tree-'));
104 const file = join(dir, 'tree');
105 const { root, cleanup } = tmpRoot();
106 try {
107 writeFileSync(file, build(CAP + 1));
108 const at = await runChild(file, CAP);
109 const under = await runChild(file, CAP - 1);
110 const over = await runChild(file, CAP + 1);
111 for (const r of [at, under]) {
112 assert.equal(r.status, 0, r.err);
113 assert.equal(r.parsed?.status, 'ok', r.text || r.err);
114 assert.ok(r.parsed.rss < RSS_BOUND_MB, `rss ${r.parsed.rss}`);
115 }
116 assert.equal(under.parsed.n, CAP - 1);
117 assert.equal(at.parsed.n, CAP);
118 assert.equal(at.parsed.first.name, '00000000');
119 assert.equal(at.parsed.last.name, String(CAP - 1).padStart(NAME, '0'));
120 assert.equal(over.status, 0, over.err);
121 assert.equal(over.parsed?.status, 'err', over.text || over.err);
122 assert.equal(over.parsed.code, 'limit_exceeded');
123 assert.equal(over.parsed.what, 'tree entries');
124 assert.equal(over.parsed.got, CAP + 1);
125 assert.equal(over.parsed.max, CAP);
126 assert.ok(over.parsed.rss < RSS_BOUND_MB, `refusal rss ${over.parsed.rss}`);
128 git(root, ['init', '--bare', 'r.git']);
129 const bare = join(root, 'r.git');
130 const slice = readFileSync(file).subarray(0, CAP * STRIDE);
131 const oid = gitBuf(bare, ['hash-object', '-w', '-t', 'tree', '--stdin'], slice).toString().trim();
132 assert.equal(git(bare, ['cat-file', '-t', oid]).trim(), 'tree');
133 const listed = gitBuf(bare, ['ls-tree', oid]).toString().trim().split('\n');
134 assert.equal(listed.length, CAP);
135 const hash = createHash('sha256');
136 for (const line of listed) {
137 const tab = line.indexOf('\t');
138 const name = line.slice(tab + 1);
139 const id = line.slice(line.indexOf(' ', line.indexOf(' ') + 1) + 1, tab);
140 hash.update(name);
141 hash.update('\0');
142 hash.update(id);
143 hash.update('\0');
144 }
145 assert.equal(hash.digest('hex'), at.parsed.digest);
146 assert.equal(listed[0].endsWith(`\t${at.parsed.first.name}`), true);
147 assert.equal(listed[0].includes(at.parsed.first.oid), true);
148 } finally {
149 rmSync(dir, { recursive: true, force: true });
150 cleanup();
151 }
152});
154// The rest of #140: listTree's result holds every entry's hex id. toHex built it by appending 20 two-digit pieces, a
155// V8 rope of ~15 cells (~480 B per id, 110 MiB at the cap, the OOM the sweep held). A flat 40-character string is
156// ~57 B. Measured in a child with --expose-gc: the heap freed by dropping many ids, per id (64 B now, 488 B as a rope).
157test('a hex id is a flat string: retaining one costs well under a rope', async () => {
158 const N = 100_000;
159 const script = `
160 import { toHex } from './src/sha1.js';
161 const src = Array.from({ length: ${N} }, (_, i) => Uint8Array.of(1, ...new Uint8Array(15), i >>> 24, (i >>> 16) & 255, (i >>> 8) & 255, i & 255));
162 const heap = () => { gc(); gc(); return process.memoryUsage().heapUsed; };
163 let held = src.map(toHex);
164 const last = held[${N} - 1];
165 const withIds = heap();
166 held = null;
167 const freed = withIds - heap();
168 process.stdout.write(String(freed / ${N}) + ' ' + last);
169 `;
170 const out = await new Promise((resolve, reject) => {
171 const child = spawn(process.execPath, ['--expose-gc', '--import', './test/ts-resolve.mjs', '--input-type=module', '-e', script], { stdio: ['ignore', 'pipe', 'inherit'] });
172 let text = '';
173 child.stdout.on('data', d => { text += d; });
174 child.on('close', status => (status === 0 ? resolve(text) : reject(new Error(`child exit ${status}`))));
175 child.on('error', reject);
176 });
177 const [perId, last] = out.split(' ');
178 assert.equal(last, `01${'00'.repeat(15)}${(N - 1).toString(16).padStart(8, '0')}`);
179 assert.ok(Number(perId) < 160, `${perId} B per retained id`);
180});
Diagnostics6 git calls · 48 ms
CallStatusSizeTime048 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=38
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2003.1 KiB · hit38 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3220
git.response.cache=HIT
git.response.duration_ms=38
git.fetch.wants=6121cf9a3a913c7e226d7524cb1b1311bed8e600
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3220
git.pkt.payload.max=3193
git.sideband.band1.bytes=3193
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2bc043a2fcfc3
Request
e71f5093…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format