GitHubivarvong

ply

main
ply6 git calls117 ms26 KiB4 cachedprotocol v2
84 lines9.0 KiBRaw
1// Build a large repo fast (no worktree: hash-object + mktree + commit-tree), then measure an edit against it.
2// node test/bigrepo.mjs [--commits 1000] [--headMB 50] [--totalGB 10] [--dir /tmp/bigrepo] [--reuse]
3import { execFileSync, spawnSync } from 'node:child_process';
4import { existsSync, mkdirSync, rmSync, statSync, readdirSync } from 'node:fs';
5import { join } from 'node:path';
6import { Rng } from './fuzz/prng.js';
7import { startGitServer } from './server.js';
8import { editFile } from '../src/edit.js';
9import { GitHttp } from '../src/client.js';
10import { Tracer, arrayExporter, flattenLine } from '../src/otel.js';
11import { unwrapAsync, unwrap } from './unwrap.js';
13const args = Object.fromEntries(process.argv.slice(2).map((a, i, arr) => a.startsWith('--') ? [a.slice(2), arr[i + 1] ?? true] : []).filter(x => x.length));
14const COMMITS = +(args.commits ?? 1000), HEAD_MB = +(args.headMB ?? 50), TOTAL_GB = +(args.totalGB ?? 10), DIR = args.dir ?? '/tmp/bigrepo';
15const bare = join(DIR, 'big.git');
16const git = (a, opt = {}) => execFileSync('git', ['-C', bare, ...a], { encoding: 'utf8', maxBuffer: 1 << 30, ...opt });
17const rng = new Rng(2024);
19if (!(args.reuse && existsSync(bare))) {
20 rmSync(DIR, { recursive: true, force: true }); mkdirSync(DIR, { recursive: true });
21 execFileSync('git', ['init', '-q', '--bare', '-b', 'main', bare]);
22 for (const [k, v] of [['uploadpack.allowFilter', 'true'], ['uploadpack.allowAnySHA1InWant', 'true'], ['http.receivepack', 'true'], ['receive.fsckObjects', 'true'], ['core.compression', '1'], ['pack.compression', '1'], ['gc.auto', '0'], ['maintenance.auto', 'false'], ['receive.autoGc', 'false']]) git(['config', k, v]);
23 const t0 = Date.now();
24 // Layout: 40 dirs × 25 files of source-ish text (~45 MB), plus a few "assets" that churn every commit (that is where the
25 // 10 GB of history comes from: each commit rewrites ~10 MB of incompressible binary).
26 const hashBlob = buf => { const r = spawnSync('git', ['-C', bare, 'hash-object', '-w', '--stdin'], { input: buf, maxBuffer: 1 << 30 }); if (r.status) throw new Error(r.stderr.toString()); return r.stdout.toString().trim(); };
27 const src = {}; // dir -> [name, oid]
28 const srcPerDir = Math.max(1, Math.round((HEAD_MB * 0.9 * 1024 * 1024) / 1000 / 1024)); // KB per source file (1000 files)
29 for (let d = 0; d < 40; d++) { src[`src/mod${String(d).padStart(2, '0')}`] = []; for (let f = 0; f < 25; f++) { const kb = Math.max(1, srcPerDir); const text = Buffer.from(Array.from({ length: Math.ceil(kb * 1024 / 48) }, (_, i) => `line ${String(i).padStart(6)} of module ${String(d).padStart(2)} file ${String(f).padStart(2)}: xxxxxxxxxx\n`).join('')); src[`src/mod${String(d).padStart(2, '0')}`].push([`file${f}.ts`, hashBlob(text)]); } }
30 const perCommitMB = (TOTAL_GB * 1024) / COMMITS; // churn per commit
31 const assetCount = 4, assetMB = perCommitMB / assetCount;
32 let parent = null, treeOidCache = null;
33 const target = { dir: 'src/mod17', name: 'file9.ts' };
34 const mktree = lines => execFileSync('git', ['-C', bare, 'mktree'], { input: lines.join('\n') + '\n', encoding: 'utf8' }).trim();
35 const subtrees = {};
36 for (const [dir, files] of Object.entries(src)) subtrees[dir] = mktree(files.map(([n, o]) => `100644 blob ${o}\t${n}`));
37 let srcTree = mktree(Object.keys(src).map(d => `040000 tree ${subtrees[d]}\t${d.split('/')[1]}`));
38 for (let c = 0; c < COMMITS; c++) {
39 // churn: new random assets each commit (incompressible → history grows ~perCommitMB)
40 const assets = []; for (let a = 0; a < assetCount; a++) assets.push([`asset${a}.bin`, hashBlob(rng.bytes(Math.floor(assetMB * 1024 * 1024)))]);
41 // the target file changes every 50 commits so its blob has real history
42 if (c % 50 === 0) { const i = src[target.dir].findIndex(([n]) => n === target.name); src[target.dir][i][1] = hashBlob(Buffer.from(`// revision ${c}\nMARK_${c}\n` + 'y'.repeat(srcPerDir * 1024))); subtrees[target.dir] = mktree(src[target.dir].map(([n, o]) => `100644 blob ${o}\t${n}`)); }
43 if (c % 50 === 0) srcTree = mktree(Object.keys(src).map(d => `040000 tree ${subtrees[d]}\t${d.split('/')[1]}`));
44 const srcT = srcTree;
45 const assetsTree = mktree(assets.map(([n, o]) => `100644 blob ${o}\t${n}`));
46 const root = mktree([`040000 tree ${srcT}\tsrc`, `040000 tree ${assetsTree}\tassets`, `100644 blob ${hashBlob(Buffer.from(`# big repo\ncommit ${c}\n`))}\tREADME.md`]);
47 const env = { ...process.env, GIT_AUTHOR_NAME: 'big', GIT_AUTHOR_EMAIL: 'b@x', GIT_COMMITTER_NAME: 'big', GIT_COMMITTER_EMAIL: 'b@x', GIT_AUTHOR_DATE: `${1700000000 + c * 60} +0000`, GIT_COMMITTER_DATE: `${1700000000 + c * 60} +0000`, GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'commit.gpgsign', GIT_CONFIG_VALUE_0: 'false' };
48 parent = execFileSync('git', ['-C', bare, 'commit-tree', root, ...(parent ? ['-p', parent] : []), '-m', `commit ${c}`], { env, encoding: 'utf8' }).trim();
49 if (c % 100 === 0) process.stderr.write(` commit ${c}/${COMMITS} ${((Date.now() - t0) / 1000) | 0}s\n`);
50 if (c % 200 === 199) git(['repack', '-d', '-q']); // keep loose-object count sane
51 }
52 git(['update-ref', 'refs/heads/main', parent]);
53 process.stderr.write(` objects written in ${((Date.now() - t0) / 1000) | 0}s; gc + bitmaps + midx...\n`);
54 const t1 = Date.now();
55 git(['-c', 'repack.writeBitmaps=true', '-c', 'pack.window=0', 'gc', '-q', '--prune=now']);
56 git(['multi-pack-index', 'write', '--bitmap']); git(['config', 'core.multiPackIndex', 'true']);
57 process.stderr.write(` gc in ${((Date.now() - t1) / 1000) | 0}s\n`);
59const du = dir => readdirSync(dir, { withFileTypes: true }).reduce((n, e) => n + (e.isDirectory() ? du(join(dir, e.name)) : statSync(join(dir, e.name)).size), 0);
60const stats = { commits: +git(['rev-list', '--count', 'main']).trim(), packBytes: du(join(bare, 'objects/pack')), headTreeEntries: git(['ls-tree', '-r', 'main']).trim().split('\n').length, headBytes: git(['ls-tree', '-r', '-l', 'main']).trim().split('\n').reduce((n, l) => n + (+l.split(/\s+/)[3] || 0), 0), bitmaps: readdirSync(join(bare, 'objects/pack')).filter(f => f.endsWith('.bitmap')) };
61console.log('repo:', JSON.stringify({ ...stats, packGB: (stats.packBytes / 2 ** 30).toFixed(2), headMB: (stats.headBytes / 2 ** 20).toFixed(1) }));
63// ---- measure ----
64const srv = await startGitServer(DIR);
65const url = `${srv.url}/big.git`;
66const head = git(['rev-parse', 'main']).trim();
67const run = async (label, fn) => { const ex = arrayExporter(); const tracer = new Tracer({ exporter: ex }); const t0 = Date.now(); let r, err; try { r = await fn(tracer); } catch (e) { err = e; } const ms = Date.now() - t0; const spans = ex.lines.flatMap(flattenLine); const root = spans.find(s => s.name === 'git.edit') ?? spans[spans.length - 1]; const http = spans.filter(s => s.attrs['http.response.body.size'] !== undefined); console.log(`${label.padEnd(44)} ${String(ms).padStart(6)}ms rtt=${http.length} wire_in=${http.reduce((n, s) => n + s.attrs['http.response.body.size'], 0)}B wire_out=${http.reduce((n, s) => n + (s.attrs['http.request.body.size'] ?? 0), 0)}B max_alloc=${root?.attrs['mem.alloc.max_single'] ?? '-'} alloc=${root?.attrs['mem.alloc.bytes'] ?? '-'}${err ? ' ERR ' + err.code + ' ' + err.message.slice(0, 60) : ''}`); return { r, spans, ms, http }; };
69const cache = new Map();
70const edit = (tracer, rev, c = new Map()) => unwrapAsync(editFile(url, { path: 'src/mod17/file9.ts', find: `MARK_${rev}`, replace: `EDITED_${rev}`, author: { name: 'e', email: 'e@x' }, message: 'edit' }, { tracer, cache: c }));
71const lastRev = Math.floor((COMMITS - 1) / 50) * 50;
72console.log('\n--- edit src/mod17/file9.ts (depth 3, 40-wide src dir, 25-wide leaf)');
73const a = await run('cold (ls-refs + commit + 3 trees + blob + push)', t => edit(t, lastRev, cache));
74console.log(' per-stage:', a.spans.filter(s => /upload-pack|receive-pack/.test(s.name)).map(s => `${s.name.replace('git.', '')}=${s.durationMs.toFixed(0)}ms/${s.attrs['http.response.body.size'] ?? s.attrs['http.request.body.size']}B`).join(' '));
75git(['update-ref', 'refs/heads/main', head]); // undo for the next run
76const b = await run('warm tree cache (same root tree → 0 tree fetches)', t => edit(t, lastRev, cache));
77git(['update-ref', 'refs/heads/main', head]);
78console.log('\n--- what the naive alternatives cost on this repo (server side, wire bytes)');
79const c = unwrap(GitHttp.create(url, { tracer: new Tracer() }));
80for (const [label, req] of [['blob:none (textbook blobless, depth 1)', { wants: [head], filter: 'blob:none', deepen: 1 }], ['tree:0 (our commit fetch)', { wants: [head], filter: 'tree:0', deepen: 1 }], ['combine:blob:none+tree:3 (one-shot trees to depth 3)', { wants: [head], filter: 'combine:blob:none+tree:3', deepen: 1 }], ['depth-1 full clone pack (what `git clone --depth 1` moves)', { wants: [head], deepen: 1 }]]) {
81 const t0 = Date.now(); let n = 0, objs = 0; try { const r = await unwrapAsync(c.fetchPack(req)); n = r.parsed.bytes; objs = r.parsed.stats.count; } catch (e) { console.log(label, 'ERR', e.message.slice(0, 80)); continue; }
82 console.log(`${label.padEnd(60)} ${String(Date.now() - t0).padStart(6)}ms ${String(n).padStart(10)}B ${objs} objects`);
84await srv.close();
Diagnostics6 git calls · 117 ms
CallStatusSizeTime0117 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2003.7 KiB108 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=108
git.response.bytes=3810
git.response.cache=MISS
git.response.duration_ms=108
git.fetch.wants=290fa37653b0f1361702ccca8217f0d3ebe760d6
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3810
git.pkt.payload.max=3783
git.sideband.band1.bytes=3783
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e13cfeacd71293
Request
c91d257e…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format