main
1// resolvePackRange on many REF_DELTAs sharing one base (#299 item 1). One blob and m identity REF deltas on it (each
2// resolves to the same blob) took 0.28 s at m = 2,000 and 15.9 s at m = 16,000 under DEFAULT_LIMITS: each delta looked
3// its base id's REF run up again with a linear walk to its end, and each delta (it rests on nothing new, its id is the
4// base's) was pushed and popped and walked that run once more. The work charged stayed 10 m + 6, so the budget never
5// refused. Oracles: node:zlib and node:crypto for the pack bytes, `git index-pack` + `git show-index` + `git hash-object`
6// for what the pack holds, and the charge model at the top of src/packrangeresolve.ts for the work, worked out by hand below.
7import { test } from 'vitest';
8import assert from 'node:assert/strict';
9import { deflateSync } from 'node:zlib';
10import { createHash } from 'node:crypto';
11import { readFileSync, writeFileSync } from 'node:fs';
12import { join } from 'node:path';
13import { scanPackIndex } from '../src/packscanindex.js';
14import { resolvePackRange, NO_BASE } from '../src/packrangeresolve.js';
15import { rangeFromBytes } from '../src/rangesource.js';
16import { git, gitBuf, tmpRoot } from './server.js';
18const cat = ps => { const o = new Uint8Array(ps.reduce((a, p) => a + p.length, 0)); let at = 0; for (const p of ps) { o.set(p, at); at += p.length; } return o; };
19// gitformat-pack: type in bits 4..6 of the first byte, size in 4 then 7-bit groups
20function hdr(type, size) { const out = []; let b = (type << 4) | (size & 15); size = Math.floor(size / 16); while (size > 0) { out.push(b | 0x80); b = size & 0x7f; size = Math.floor(size / 128); } out.push(b); return Uint8Array.from(out); }
21const data = new TextEncoder().encode('hello\n');
22const oid = createHash('sha1').update(`blob ${data.length}\0`).update(data).digest();
23const oidHex = Buffer.from(oid).toString('hex');
24// gitformat-pack delta: source size 6, result size 6, copy offset 0 size 6 (0x90: size byte 0 present) = identity
25const delta = Uint8Array.from([6, 6, 0x90, 6]);
26function sharedPack(m) {
27 const parts = [Uint8Array.from([0x50, 0x41, 0x43, 0x4b, 0, 0, 0, 2, ((m + 1) >>> 24) & 255, ((m + 1) >>> 16) & 255, ((m + 1) >>> 8) & 255, (m + 1) & 255]), hdr(3, 6), deflateSync(data)];
28 const body = deflateSync(delta);
29 for (let i = 0; i < m; i++) parts.push(hdr(7, delta.length), oid, body);
30 const all = cat(parts);
31 return cat([all, createHash('sha1').update(all).digest()]);
32}
33const streamOf = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
34async function resolve(pack) {
35 const idx = await scanPackIndex(streamOf(pack));
36 assert.ok(idx.ok, idx.ok ? '' : idx.error.message);
37 return resolvePackRange(idx.value, rangeFromBytes(pack));
38}
40test('m REF deltas on one base resolve to what git index-pack makes of the pack', async () => {
41 const m = 64;
42 const pack = sharedPack(m);
43 const { root, cleanup } = tmpRoot();
44 try {
45 const repo = join(root, 'r.git');
46 git(root, ['init', '-q', '--bare', repo]);
47 const file = join(repo, 'objects', 'pack', 'pack-shared.pack');
48 writeFileSync(file, pack);
49 // index-pack takes a pack holding one object many times (verify-pack refuses it, "appears twice")
50 git(repo, ['index-pack', file]);
51 // show-index: offset sha1 (crc), one line per entry
52 const byOffset = gitBuf(repo, ['show-index'], readFileSync(file.replace(/\.pack$/u, '.idx'))).toString().trim().split('\n').map(l => l.split(' ')).toSorted((a, b) => Number(a[0]) - Number(b[0]));
53 assert.equal(byOffset.length, m + 1);
54 assert.equal(gitBuf(root, ['hash-object', '--stdin'], data).toString().trim(), oidHex);
55 const r = await resolve(pack);
56 assert.ok(r.ok, r.ok ? '' : r.error.message);
57 assert.equal(r.value.count, m + 1);
58 for (let i = 0; i <= m; i++) {
59 assert.equal(r.value.oid(i).value, byOffset[i][1], `entry ${i} oid`);
60 assert.equal(r.value.type(i).value, 3, `entry ${i} type`);
61 // by hand: every delta names the blob (entry 0) by id
62 assert.equal(r.value.columns.depth[i], i === 0 ? 0 : 1, `entry ${i} depth`);
63 assert.equal(r.value.columns.base[i], i === 0 ? NO_BASE : 0, `entry ${i} base`);
64 assert.equal(r.value.columns.size[i], 6, `entry ${i} size`);
65 }
66 assert.equal(byOffset[0][1], oidHex);
67 } finally {
68 cleanup();
69 }
70});
72test('the work charged for m shared-base REF deltas is 11 m + 6, worked out by hand', async () => {
73 // the blob inflated once (6); each delta inflated (4) and applied (6), and visited once as a child of its base (1)
74 for (const m of [1, 10, 1000]) {
75 const r = await resolve(sharedPack(m));
76 assert.ok(r.ok, r.ok ? '' : r.error.message);
77 assert.equal(r.value.stats.work, 11 * m + 6, `m=${m}`);
78 }
79});
81// Wall time under CPU contention (CI shards run files side by side): the two sizes are timed interleaved, best of 7 each,
82// and the ratio is taken up to 3 times, judged on the best attempt, as test/linear-matchers-274.test.js does. A quadratic
83// resolve measures about 13 x here however many retries it gets; the linear one about 3.3 to 4.6 x.
84test('resolving m REF deltas on one base is linear-ish: 4 x the deltas take at most 6 x the time', { timeout: 120_000 }, async () => {
85 const prep = async m => {
86 const pack = sharedPack(m);
87 const idx = await scanPackIndex(streamOf(pack));
88 assert.ok(idx.ok);
89 return { idx: idx.value, pack };
90 };
91 const once = async ({ idx, pack }) => {
92 const t0 = performance.now();
93 const r = await resolvePackRange(idx, rangeFromBytes(pack));
94 const ms = performance.now() - t0;
95 assert.ok(r.ok);
96 return ms;
97 };
98 const warm = await prep(500);
99 const smallIn = await prep(2000);
100 const bigIn = await prep(8000);
101 await once(warm);
102 const attempt = async () => {
103 let small = Infinity;
104 let big = Infinity;
105 for (let k = 0; k < 7; k++) {
106 // oxlint-disable-next-line no-await-in-loop -- timed runs, in turn
107 small = Math.min(small, await once(smallIn));
108 // oxlint-disable-next-line no-await-in-loop -- timed runs, in turn
109 big = Math.min(big, await once(bigIn));
110 }
111 return { small, big, ratio: big / small };
112 };
113 let best = { small: 0, big: 0, ratio: Infinity };
114 for (let a = 0; a < 3 && best.ratio > 6; a++) {
115 // oxlint-disable-next-line no-await-in-loop -- attempts, in turn
116 const r = await attempt();
117 if (r.ratio < best.ratio) best = r;
118 }
119 assert.ok(best.ratio <= 6, `m=2,000 ${best.small.toFixed(0)} ms, m=8,000 ${best.big.toFixed(0)} ms: ratio ${best.ratio.toFixed(2)}`);
120});
Diagnostics4 git calls · 397 ms
CallStatusSizeTime0397 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=UPDATING git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit7 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=387 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=7 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2002.7 KiB387 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=2730 git.response.cache=MISS git.response.duration_ms=387 git.fetch.wants=d43b1426549ad83b573a2cee57e99358d882f3f3 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2730 git.pkt.payload.max=2709 git.sideband.band1.bytes=2708 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e21191de3ec6f8
- Request
- 1a1fca6e…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format