main
1// describePack against packs git wrote. verify-pack -v is the row oracle; index-pack --fix-thin is the thin-base oracle.
2// A limit probe uses git's own columns for the boundary (header size, chain depth, cat-file size of each delta).
3import { test } from 'vitest';
4import assert from 'node:assert/strict';
5import { execFileSync } from 'node:child_process';
6import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
7import { tmpdir } from 'node:os';
8import { join } from 'node:path';
9import { describePack } from '../../src/pack.js';
10import { TYPE, TYPE_ID } from '../../src/objects.js';
11import { toHex } from '../../src/sha1.js';
12import { DEFAULT_LIMITS, LIMIT_OBJECT_OVERHEAD, withLimits } from '../../src/limits.js';
13import { PACK_TRAILER_BYTES, PACK_VERSION } from '../../src/format.js';
14import { CONTENTS, COMPRESSION, OFS, DEPTH, THIN, PACK_CELLS, LIMIT_PROBES, CELL_COUNT, REPRESENTATIVE } from './describe-pack.space.js';
16const ENV = {
17 PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR ?? '/tmp',
18 GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', GIT_PAGER: 'cat', GIT_TERMINAL_PROMPT: '0', LC_ALL: 'C.UTF-8',
19};
20const ROW = /^([0-9a-f]{40})\s+(commit|tree|blob|tag)\s+(\d+)\s+(\d+)\s+(\d+)(?:\s+(\d+)\s+([0-9a-f]{40}))?$/;
21const NON = /^non delta: (\d+) objects?$/;
22const CHAIN = /^chain length = (\d+): (\d+) objects?$/;
24const git = (cwd, args, input) => execFileSync('git', args, { cwd, env: ENV, input, stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 1 << 26 });
25const gitText = (cwd, args, input) => git(cwd, args, input).toString('utf8');
26const commit = (repo, message) => {
27 git(repo, ['add', '-A']);
28 git(repo, ['-c', 'user.name=g', '-c', 'user.email=g@x', 'commit', '-q', '-m', message]);
29};
30const file = (repo, name, n, mark) => {
31 const b = Buffer.alloc(n, 0x41);
32 if (mark !== undefined) b.set(Buffer.from(mark), 0);
33 writeFileSync(join(repo, name), b);
34};
36function makeRepo(root, content) {
37 const repo = join(root, content);
38 mkdirSync(repo);
39 git(repo, ['init', '-q', '-b', 'main']);
40 if (content === 'blob' || content === 'commit') {
41 file(repo, 'a.txt', 32, 'a');
42 commit(repo, 'c');
43 } else if (content === 'tag') {
44 file(repo, 'a.txt', 2000, 'a');
45 commit(repo, 'base');
46 file(repo, 'a.txt', 2000, 'b');
47 commit(repo, 'tip');
48 git(repo, ['-c', 'user.name=g', '-c', 'user.email=g@x', 'tag', '-a', 'v1', '-m', 'annotated']);
49 } else if (content === 'history') {
50 for (let i = 0; i < 6; i++) {
51 file(repo, 'a.txt', 2000, `h${i}`);
52 commit(repo, `c${i}`);
53 }
54 } else if (content === 'fan') {
55 for (let i = 0; i < 4; i++) file(repo, `f${i}.txt`, 1500, `f${i}`);
56 commit(repo, 'base');
57 for (let i = 0; i < 4; i++) file(repo, `f${i}.txt`, 1500, `g${i}`);
58 commit(repo, 'tip');
59 } else throw new Error(content);
60 return repo;
61}
63function packInput(repo, content, thin) {
64 if (content === 'blob') return { args: [], input: `${gitText(repo, ['rev-parse', 'HEAD:a.txt']).trim()}\n` };
65 const revs = ['--revs'];
66 if (content === 'tag') return { args: revs, input: thin ? 'refs/tags/v1\n^HEAD~1\n' : 'refs/tags/v1\n' };
67 if (thin && content !== 'commit') return { args: revs, input: 'HEAD\n^HEAD~1\n' };
68 return { args: revs, input: 'HEAD\n' };
69}
71function parseVerify(text) {
72 const rows = [];
73 let non = null;
74 const chains = new Map();
75 for (const line of text.split('\n')) {
76 const t = line.trim();
77 const row = ROW.exec(t);
78 if (row) {
79 rows.push({
80 oid: row[1], type: row[2], size: Number(row[3]), packed: Number(row[4]), offset: Number(row[5]),
81 depth: row[6] ? Number(row[6]) : 0, base: row[7] ?? null,
82 });
83 continue;
84 }
85 const nd = NON.exec(t);
86 if (nd) non = Number(nd[1]);
87 const ch = CHAIN.exec(t);
88 if (ch) chains.set(Number(ch[1]), Number(ch[2]));
89 }
90 assert.notEqual(non, null, text);
91 let gotNon = 0;
92 const gotChains = new Map();
93 for (const r of rows) {
94 if (r.depth === 0) gotNon++;
95 else gotChains.set(r.depth, (gotChains.get(r.depth) ?? 0) + 1);
96 }
97 assert.equal(non, gotNon, text);
98 assert.equal(gotChains.size, chains.size, text);
99 for (const [d, n] of gotChains) assert.equal(chains.get(d), n, `chain ${d}`);
100 return rows;
101}
103function eqRow(o, row, label) {
104 assert.equal(o.size, row.size, `${label} size`);
105 assert.equal(o.headerBytes + o.compressed, row.packed, `${label} size-in-pack`);
106 assert.equal(o.offset, row.offset, `${label} offset`);
107 assert.equal(o.depth, row.depth, `${label} depth`);
108 if (row.depth === 0) assert.equal(o.wireType, TYPE_ID[row.type], `${label} wire`);
109 else assert.ok(o.wireType === TYPE.OFS_DELTA || o.wireType === TYPE.REF_DELTA, `${label} delta wire ${o.wireType}`);
110 if (o.oid === null) {
111 assert.equal(o.type, null, label);
112 assert.ok(o.depth >= 1, label);
113 } else {
114 assert.equal(o.oid, row.oid, `${label} oid`);
115 assert.equal(TYPE_ID[row.type], o.type, `${label} type`);
116 }
117 if (row.base === null) assert.deepEqual(o.base, { tag: 'none' }, `${label} base`);
118 else if (o.base.tag === 'ofs') {
119 if (o.base.oid !== null) assert.equal(o.base.oid, row.base, `${label} ofs base`);
120 else assert.equal(o.oid, null, label);
121 } else assert.equal(o.base.tag === 'ref' ? o.base.oid : null, row.base, `${label} ref base`);
122}
124const hashOf = text => {
125 const m = text.match(/[0-9a-f]{40}/g);
126 assert.ok(m, text);
127 return m[m.length - 1];
128};
130function verifySelf(scratch, bytes, layout) {
131 const packPath = join(scratch, 'p.pack');
132 const idxPath = join(scratch, 'p.idx');
133 rmSync(packPath, { force: true });
134 rmSync(idxPath, { force: true });
135 writeFileSync(packPath, bytes);
136 const sum = hashOf(gitText(scratch, ['index-pack', packPath]));
137 assert.equal(layout.trailer, sum);
138 const rows = parseVerify(gitText(scratch, ['verify-pack', '-v', packPath]));
139 assert.equal(layout.count, rows.length);
140 assert.equal(layout.objects.length, rows.length);
141 assert.deepEqual(layout.missingBases, []);
142 const byOff = new Map(rows.map(r => [r.offset, r]));
143 for (const o of layout.objects) eqRow(o, byOff.get(o.offset), `off ${o.offset}`);
144 return rows;
145}
147function verifyThin(repo, bytes, layout) {
148 let bare;
149 try { bare = gitText(repo, ['index-pack', '--stdin'], bytes); } catch (e) { bare = `${e.stderr ?? e.message}`; }
150 assert.match(bare, /unresolved delta/);
151 const fixedHash = hashOf(gitText(repo, ['index-pack', '--stdin', '--fix-thin'], bytes));
152 const fixedPath = join(repo, '.git', 'objects', 'pack', `pack-${fixedHash}.pack`);
153 const rows = parseVerify(gitText(repo, ['verify-pack', '-v', fixedPath]));
154 const cut = bytes.length - PACK_TRAILER_BYTES;
155 const appended = rows.filter(r => r.offset >= cut);
156 const kept = rows.filter(r => r.offset < cut);
157 // git 2.43 appends those bases in the same order describePack reports: first-seen in pack order.
158 assert.deepEqual(layout.missingBases, appended.map(r => r.oid));
159 assert.equal(layout.objects.length, kept.length);
160 const byOff = new Map(kept.map(r => [r.offset, r]));
161 for (const o of layout.objects) eqRow(o, byOff.get(o.offset), `thin off ${o.offset}`);
162 assert.equal(layout.trailer, toHex(bytes.subarray(bytes.length - PACK_TRAILER_BYTES)));
163 return kept;
164}
166test('describePack matches git verify-pack over the pack product and at every limit boundary', { timeout: 180_000 }, () => {
167 const root = mkdtempSync(join(tmpdir(), 'ply-describe-'));
168 const scratch = join(root, 'scratch');
169 mkdirSync(scratch);
170 git(scratch, ['init', '-q']);
171 const repos = new Map(CONTENTS.map(c => [c, makeRepo(root, c)]));
172 let packs = 0;
173 let thinHits = 0;
174 let multiBase = 0;
175 let rep = null;
176 const built = [];
177 for (const content of CONTENTS) for (const compression of COMPRESSION) for (const ofs of OFS) for (const depth of DEPTH) for (const thin of THIN) {
178 const repo = repos.get(content);
179 const { args, input } = packInput(repo, content, thin);
180 const window = depth === 0 ? 0 : depth;
181 const cmd = ['-c', `pack.compression=${compression}`, '-c', 'pack.threads=1', 'pack-objects', '--stdout', `--depth=${depth}`, `--window=${window}`, ...args];
182 if (ofs) cmd.push('--delta-base-offset');
183 if (thin) cmd.push('--thin');
184 built.push({ content, compression, ofs, depth, thin, repo, bytes: new Uint8Array(git(repo, cmd, input)) });
185 }
186 try {
187 for (const cell of built) {
188 const { content, compression, ofs, depth, thin, repo, bytes } = cell;
189 const described = describePack(bytes);
190 const label = `${content} c${compression} ofs${ofs} d${depth} thin${thin}`;
191 assert.equal(described.ok, true, label);
192 if (!described.ok) continue;
193 const layout = described.value;
194 assert.equal(layout.version, PACK_VERSION, label);
195 assert.equal(layout.count, layout.objects.length, label);
196 let rows;
197 if (layout.missingBases.length === 0) rows = verifySelf(scratch, bytes, layout);
198 else {
199 assert.equal(thin, true, `${label} produced a thin base without --thin`);
200 rows = verifyThin(repo, bytes, layout);
201 thinHits++;
202 if (layout.missingBases.length >= 2) multiBase++;
203 }
204 packs++;
205 if (content === REPRESENTATIVE.content && compression === REPRESENTATIVE.compression && ofs === REPRESENTATIVE.ofs && depth === REPRESENTATIVE.depth && thin === REPRESENTATIVE.thin) {
206 rep = { repo, bytes, rows };
207 }
208 }
209 assert.equal(packs, PACK_CELLS);
210 assert.ok(thinHits > 0, 'no cell was a thin pack');
211 assert.ok(multiBase > 0, 'no thin pack had two missing bases');
212 assert.ok(rep, 'representative pack was not built');
213 const maxHeader = Math.max(...rep.rows.map(r => r.size));
214 const maxDepth = Math.max(...rep.rows.map(r => r.depth));
215 const deltaSizes = rep.rows.filter(r => r.depth > 0).map(r => Number(gitText(rep.repo, ['cat-file', '-s', r.oid]).trim()));
216 assert.ok(maxDepth >= 1 && deltaSizes.length > 0, 'representative pack has no delta');
217 const headerSum = rep.rows.reduce((n, r) => n + r.size, 0);
218 const maxDelta = Math.max(...deltaSizes);
219 const bounds = [
220 ['maxPackObjects', rep.rows.length, 'pack objects'],
221 ['maxObjectSize', maxHeader, 'pack object size'],
222 ['maxInflatedTotal', headerSum + rep.rows.length * LIMIT_OBJECT_OVERHEAD + deltaSizes.reduce((n, s) => n + s, 0), 'delta result size'],
223 ['maxDeltaChain', maxDepth, 'delta chain depth'],
224 ['maxDeltaResult', maxDelta, 'delta result size'],
225 ];
226 let probes = 0;
227 for (const [name, boundary, what] of bounds) {
228 assert.equal(describePack(rep.bytes, withLimits({ [name]: boundary })).ok, true, `${name}=${boundary}`);
229 const under = describePack(rep.bytes, withLimits({ [name]: boundary - 1 }));
230 assert.equal(under.ok, false, `${name}=${boundary - 1}`);
231 if (!under.ok) {
232 assert.equal(under.error.code, 'limit_exceeded');
233 assert.equal(under.error.what, what, `${name} refused as ${under.error.what}`);
234 }
235 probes += 2;
236 }
237 const nan = describePack(rep.bytes, withLimits({ maxPackObjects: Number.NaN }));
238 assert.equal(nan.ok, false);
239 if (!nan.ok) {
240 assert.equal(nan.error.code, 'bad_request');
241 assert.equal(nan.error.what, 'maxPackObjects');
242 }
243 // a key left out is the default (#156 parseLimits); given as null it is refused
244 const missing = { ...DEFAULT_LIMITS };
245 delete missing.maxDeltaChain;
246 assert.deepEqual(describePack(rep.bytes, missing), describePack(rep.bytes, DEFAULT_LIMITS));
247 const gone = describePack(rep.bytes, { ...DEFAULT_LIMITS, maxDeltaChain: null });
248 assert.equal(gone.ok, false);
249 if (!gone.ok) {
250 assert.equal(gone.error.code, 'bad_request');
251 assert.equal(gone.error.what, 'maxDeltaChain');
252 }
253 probes += 2;
254 assert.equal(probes, LIMIT_PROBES);
255 assert.equal(packs + probes, CELL_COUNT);
256 console.log(`describe-pack: ${packs} packs, ${thinHits} thin (${multiBase} with two or more bases), ${probes} limit probes`);
257 } finally {
258 rmSync(root, { recursive: true, force: true });
259 }
260});
Diagnostics4 git calls · 223 ms
CallStatusSizeTime0223 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2004.0 KiB218 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=217 git.response.bytes=4126 git.response.cache=MISS git.response.duration_ms=218 git.fetch.wants=ab8a928e559613185b60196c6c76d4c179d90d7d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4126 git.pkt.payload.max=4105 git.sideband.band1.bytes=4104 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e31178c937bdc9
- Request
- 7de49777…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format