GitHubivarvong
ply
main
1// Honest history under both presets (owner decision 1 of the resolve work ceiling): this repository's own pack, built
2// read-only by `git pack-objects --revs --stdout` from HEAD, resolves under DEFAULT_LIMITS and under WORKER_LIMITS, and
3// every object's oid, type and size is what `git index-pack` + `git verify-pack -v` say of the same bytes.
4//
5// Measured on fbbf5d2 (node 24, git 2.50.0), work = bytes inflated + delta results made, the same under both presets:
6// this repository, HEAD history (this test) 3,872 objects (2,985 deltas), 3,307,891 pack bytes, work 69,665,726
7// this repository, every ref (the reviewer's) 7,541 objects, 7,852,523 pack bytes, work 109,854,403
8// an 18 MB pack of this repository, the largest real pack in the session scratch (a clone's own packfile, mostly
9// whole objects) 3,500 objects, 18,028,461 pack bytes, work 4,734,753
10// all ok under both presets, against ceilings of 2,147,483,648 (DEFAULT_LIMITS.maxResolveWork) and 536,870,912
11// (WORKER_LIMITS.maxResolveWork). No clone of git.git or another larger real repository was at hand, so nothing larger
12// was measured. Work counts what resolving reads back: deltas and the bases they rest on, not whole objects nothing
13// rests on, so it is at least the bytes of the objects git stored as deltas (62,663,356 here).
14// A shallow checkout packs less; the oracle is git on the same bytes, so the test holds for whatever HEAD reaches.
15import { test } from 'vitest';
16import assert from 'node:assert/strict';
17import { execFileSync } from 'node:child_process';
18import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
19import { join } from 'node:path';
20import { tmpdir } from 'node:os';
21import { scanPackIndex } from '../src/packscanindex.js';
22import { resolvePackRange } from '../src/packrangeresolve.js';
23import { rangeFromBytes } from '../src/rangesource.js';
24import { DEFAULT_LIMITS, WORKER_LIMITS } from '../src/limits.js';
26const TYPE = { commit: 1, tree: 2, blob: 3, tag: 4 };
27const streamOf = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
29test('this repository\'s own history resolves under DEFAULT_LIMITS and WORKER_LIMITS to the objects git finds in the same pack', { timeout: 600_000 }, async () => {
30 const pack = new Uint8Array(execFileSync('git', ['pack-objects', '--revs', '--stdout', '--delta-base-offset', '-q'], { input: 'HEAD\n', maxBuffer: 1 << 30 }));
31 const dir = mkdtempSync(join(tmpdir(), 'ply-resolve-honest-'));
32 let inPack;
33 try {
34 writeFileSync(join(dir, 'p.pack'), pack);
35 execFileSync('git', ['index-pack', '-o', join(dir, 'p.idx'), join(dir, 'p.pack')]);
36 const listing = execFileSync('git', ['verify-pack', '-v', join(dir, 'p.idx')], { encoding: 'utf8', maxBuffer: 1 << 30 });
37 // "<oid> <type> <size> <bytes in pack> <offset> [<depth> <base>]": one line per object, the last two on a delta.
38 // The size there is the delta's own, so each object's size is asked of the repository below.
39 inPack = [...listing.matchAll(/^([0-9a-f]{40}) (?:commit|tree|blob|tag) +\d+ \d+ \d+( \d+ [0-9a-f]{40})?$/gmu)].map(m => ({ oid: m[1], delta: m[2] !== undefined }));
40 } finally { rmSync(dir, { recursive: true, force: true }); }
41 assert.equal(inPack.length, new DataView(pack.buffer, pack.byteOffset).getUint32(8), 'git listed every object the pack header counts');
42 const sized = execFileSync('git', ['cat-file', '--batch-check=%(objectname) %(objecttype) %(objectsize)'], { input: `${inPack.map(o => o.oid).join('\n')}\n`, encoding: 'utf8', maxBuffer: 1 << 30 }).trim().split('\n');
43 const want = sized.map(line => { const [oid, type, size] = line.split(' '); return { oid, type: TYPE[type], size: Number(size) }; });
44 assert.equal(want.length, inPack.length);
45 const key = o => `${o.oid} ${o.type} ${o.size}`;
46 const wantKeys = want.map(key).sort();
47 // every delta's result is made at least once, so the work is at least the bytes of the objects git stored as deltas
48 const deltaBytes = want.reduce((a, o, i) => a + (inPack[i].delta ? o.size : 0), 0);
49 for (const [name, limits] of [['DEFAULT_LIMITS', DEFAULT_LIMITS], ['WORKER_LIMITS', WORKER_LIMITS]]) {
50 const idx = await scanPackIndex(streamOf(pack), { limits });
51 assert.equal(idx.ok, true, idx.ok ? '' : `${name} scan: ${idx.error.message}`);
52 const r = await resolvePackRange(idx.value, rangeFromBytes(pack), { limits });
53 assert.equal(r.ok, true, r.ok ? '' : `${name}: ${r.error.code}: ${r.error.message}`);
54 const got = Array.from({ length: r.value.count }, (_, i) => key({ oid: r.value.oid(i).value, type: r.value.type(i).value, size: r.value.columns.size[i] }));
55 assert.deepEqual(got.sort(), wantKeys, `${name}: oid, type and size of every object`);
56 assert.ok(r.value.stats.work >= deltaBytes, `${name}: work ${r.value.stats.work} under the ${deltaBytes} bytes of deltified objects`);
57 assert.ok(r.value.stats.work <= limits.maxResolveWork, `${name}: work ${r.value.stats.work} past the ceiling`);
58 console.log(JSON.stringify({ preset: name, objects: r.value.count, packBytes: pack.length, deltas: inPack.filter(o => o.delta).length, deltaBytes, work: r.value.stats.work, workMax: r.value.stats.workMax, ceiling: limits.maxResolveWork }));
59 }
60});
Diagnostics6 git calls · 126 ms
CallStatusSizeTime0126 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=11990 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetch2002.2 KiB117 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=117 git.response.bytes=2293 git.response.cache=MISS git.response.duration_ms=117 git.fetch.wants=74dbc563ef228656322c264344cdd649bffebdf3 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2293 git.pkt.payload.max=2266 git.sideband.band1.bytes=2266 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e1cbdeeb12c6f8
- Request
- 8fbf9265…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format