main
1// Generate a repo with a branching/merging history and churned files (so packs have delta chains),
2// then read the full object model back from git. The model is the oracle for expected pack contents.
3import { join } from 'node:path';
4import { mkdirSync, writeFileSync } from 'node:fs';
5import { git, gitBuf, initRepo } from '../server.js';
7// files: a set of paths at varying depth; each commit edits some, adds some, deletes some.
8export function genDag(rng, work, { commits = 14, maxParents = 3, branches = 3 } = {}) {
9 initRepo(work);
10 const paths = ['a.txt', 'big.bin', 'd1/x.txt', 'd1/y.txt', 'd1/d2/z.txt', 'd1/d2/d3/w.txt', 'd1/d2/d3/d4/v.txt', 'e/f.txt'];
11 const content = Object.fromEntries(paths.map(p => [p, Buffer.from(`${p}\n` + 'line\n'.repeat(rng.int(1, 40)))]));
12 content['big.bin'] = Buffer.alloc(rng.int(2000, 9000), rng.int(0, 255));
13 const heads = []; // commit oids usable as parents
14 const env = { ...process.env, GIT_AUTHOR_NAME: 'dag', GIT_AUTHOR_EMAIL: 'd@x', GIT_COMMITTER_NAME: 'dag', GIT_COMMITTER_EMAIL: 'd@x', };
15 const commitOids = [];
16 for (let i = 0; i < commits; i++) {
17 // mutate working tree
18 for (const p of paths) {
19 if (rng.bool(0.5)) { const b = content[p]; const o = rng.int(0, b.length); content[p] = Buffer.concat([b.subarray(0, o), Buffer.from(rng.bool(0.5) ? `edit${i}\n` : ''), b.subarray(Math.min(b.length, o + rng.int(0, 10)))]); }
20 mkdirSync(join(work, p, '..'), { recursive: true });
21 writeFileSync(join(work, p), content[p]);
22 }
23 if (rng.bool(0.3)) { const np = `new${i}/n.txt`; mkdirSync(join(work, np, '..'), { recursive: true }); writeFileSync(join(work, np), `n${i}`); paths.push(np); content[np] = Buffer.from(`n${i}`); }
24 git(work, ['add', '-A']);
25 const tree = git(work, ['write-tree']).trim();
26 const nParents = i === 0 ? 0 : Math.min(heads.length, rng.weighted([[1, 6], [2, 2], [3, 1]]));
27 const parents = [];
28 while (parents.length < nParents) { const p = rng.pick(heads); if (!parents.includes(p)) parents.push(p); }
29 const date = `${1700000000 + i * 1000} +0000`;
30 const oid = git(work, ['commit-tree', tree, ...parents.flatMap(p => ['-p', p]), '-m', `c${i}`], { env: { ...env, GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date } }).trim();
31 commitOids.push(oid);
32 heads.push(oid);
33 if (heads.length > branches) heads.splice(rng.int(0, heads.length - 1), 1);
34 }
35 const tip = commitOids[commitOids.length - 1];
36 git(work, ['update-ref', 'refs/heads/main', tip]);
37 heads.forEach((h, i) => git(work, ['update-ref', `refs/heads/b${i}`, h]));
38 git(work, ['tag', '-a', 'v1', '-m', 'v1', commitOids[Math.floor(commits / 2)]], { env: { ...env, GIT_COMMITTER_DATE: '1700000000 +0000' } });
39 return { commitOids, tip, branchHeads: heads };
40}
42// Read the model back from git: commits (parents, tree), every tree's entries, blob sizes, depths.
43export function readModel(repo) {
44 const type = new Map(), size = new Map();
45 for (const l of git(repo, ['cat-file', '--batch-check', '--batch-all-objects']).trim().split('\n')) { const [o, t, s] = l.split(' '); type.set(o, t); size.set(o, +s); }
46 const commits = new Map();
47 for (const l of git(repo, ['rev-list', '--all', '--parents']).trim().split('\n')) { const [c, ...ps] = l.split(' '); commits.set(c, { parents: ps, tree: git(repo, ['rev-parse', `${c}^{tree}`]).trim(), date: +git(repo, ['log', '-1', '--format=%ct', c]).trim() }); }
48 const refs = new Map(git(repo, ['for-each-ref', '--format=%(refname) %(objectname)']).trim().split('\n').map(l => l.split(' ')).map(([r, o]) => [r, o]));
49 const trees = new Map();
50 for (const [o, t] of type) if (t === 'tree') trees.set(o, git(repo, ['ls-tree', o]).trim().split('\n').filter(Boolean).map(l => { const [m, ty, oid, ...n] = l.split(/\s+/); return { mode: m, type: ty, oid, name: n.join(' ') }; }));
51 const tags = new Map();
52 for (const [o, t] of type) if (t === 'tag') tags.set(o, git(repo, ['rev-parse', `${o}^{}`]).trim());
53 return { type, size, commits, trees, tags, refs };
54}
56// ---- pure set computations over the model ----
57// upload-pack do_got_oid(): a have marks its direct parents THEY_HAVE; a later have already marked is not added.
58// Order-dependent by design in git; we reproduce it rather than idealize it.
59export function effectiveHaves(model, haves) {
60 const theyHave = new Set(), out = [];
61 for (const h of haves) {
62 if (!model.type.has(h)) continue;
63 const knew = theyHave.has(h);
64 theyHave.add(h);
65 // parents are marked even for an already-known have (captured from upload-pack via uploadpack.packObjectsHook)
66 if (model.commits.has(h)) for (const p of model.commits.get(h).parents) theyHave.add(p);
67 if (!knew) out.push(h);
68 }
69 return out;
70}
71export function reachableCommits(model, roots, { depth = Infinity, since = 0, exclude = new Set() } = {}) {
72 // BFS by generation from each root; depth 1 = root only. `since`: parents older than it are cut.
73 // `exclude`: commits reachable from deepen-not refs are cut. Returns Map commit → minDepth, and boundary set.
74 const seen = new Map();
75 let frontier = roots.filter(r => model.commits.has(r)).map(r => [r, 1]);
76 const boundary = new Set();
77 while (frontier.length) {
78 const next = [];
79 for (const [c, d] of frontier) {
80 if (seen.has(c) || exclude.has(c)) continue;
81 seen.set(c, d);
82 const ps = model.commits.get(c).parents;
83 const kept = ps.filter(p => !exclude.has(p) && model.commits.get(p).date >= since);
84 // git: a commit at exactly the cutoff depth is reported shallow even if it has no parents (observed on root commits)
85 if (d >= depth) { boundary.add(c); continue; }
86 if (kept.length < ps.length) boundary.add(c);
87 for (const p of kept) next.push([p, d + 1]);
88 }
89 frontier = next;
90 }
91 return { commits: seen, boundary };
92}
94// Objects (trees+blobs) of a set of trees with their min depth from root (root tree depth 0).
95export function treeClosure(model, rootTrees) {
96 const depthOf = new Map(); // oid → min depth
97 const stack = rootTrees.map(t => [t, 0]);
98 while (stack.length) {
99 const [t, d] = stack.pop();
100 if (depthOf.has(t) && depthOf.get(t) <= d) continue;
101 depthOf.set(t, d);
102 for (const e of model.trees.get(t) ?? []) {
103 if (e.type === 'tree') stack.push([e.oid, d + 1]);
104 else if (e.type === 'blob') { if (!depthOf.has(e.oid) || depthOf.get(e.oid) > d + 1) depthOf.set(e.oid, d + 1); }
105 }
106 }
107 return depthOf;
108}
110export function parseFilter(f) {
111 if (!f) return () => true;
112 if (f.startsWith('combine:')) { const fs = f.slice(8).split('+').map(parseFilter); return (o, m) => fs.every(x => x(o, m)); }
113 if (f === 'blob:none') return o => o.type !== 'blob';
114 if (f.startsWith('blob:limit=')) { const n = parseInt(f.slice(11), 10); return o => o.type !== 'blob' || o.size < n; }
115 if (f.startsWith('tree:')) { const n = parseInt(f.slice(5), 10); return o => (o.type !== 'blob' && o.type !== 'tree') || o.depth < n; }
116 throw new Error(`unknown filter ${f}`);
117}
118// pack-objects can serve a request from the bitmap index only for these filters (git 2.43 pack-bitmap.c filter_bitmap)
119export function bitmapSupportsFilter(f) {
120 if (!f) return true;
121 if (f.startsWith('combine:')) return f.slice(8).split('+').every(bitmapSupportsFilter);
122 return f === 'blob:none' || f.startsWith('blob:limit=') || f === 'tree:0';
123}
125// Expected object set for a v2 fetch.
126// mode 'bitmap' : exact reachable(wants) \ reachable(haves) — server has a bitmap, no deepen, supported filter
127// mode 'traversal' : git's revision walk with edge-based tree marking (aggressive when the request is shallow)
128export function expectedObjects(model, { wants, haves = [], deepen, deepenSince, deepenNot = [], filter, shallow = [], bitmap = false, thinPack = false }) {
129 const peel = o => (model.tags.has(o) ? model.tags.get(o) : o);
130 const exclude = new Set();
131 for (const r of deepenNot) { const o = model.refs.get(r); if (!o) continue; for (const c of reachableCommits(model, [peel(o)]).commits.keys()) exclude.add(c); }
132 const deepening = deepen !== undefined || !!deepenSince || deepenNot.length > 0;
133 const mode = bitmap && !deepening && bitmapSupportsFilter(filter) ? 'bitmap' : 'traversal';
134 const eff = effectiveHaves(model, haves);
135 const knownHaves = eff.map(peel).filter(h => model.commits.has(h));
136 const nonCommitHaves = eff.filter(h => !model.commits.has(h) && !model.tags.has(h));
137 const clientShallow = new Set(shallow.filter(c => model.commits.has(c)));
139 // 1. depth walk from wants (independent of haves): boundary = commits at the cutoff; they are shallow for this request
140 let wantCommits = wants.map(peel).filter(w => model.commits.has(w));
141 const walk0 = reachableCommits(model, wantCommits, { depth: deepen ?? Infinity, since: deepenSince ?? 0, exclude });
142 // send_unshallow(): a client shallow inside the window that is not on the boundary is unshallowed; its parents become wants
143 const unshallowed = deepening ? [...clientShallow].filter(c => walk0.commits.has(c) && !walk0.boundary.has(c)) : [];
144 wantCommits = [...wantCommits, ...unshallowed.flatMap(u => model.commits.get(u).parents)];
145 // the shallow set was computed before the parents were added: they get no fresh depth budget (they lie inside the window)
146 const { commits: inDepth, boundary } = walk0;
147 const cut = new Set([...clientShallow, ...walk0.boundary]);
149 // 2. have closure: ancestors of haves, not descending past cut commits. An unshallowed client commit is an
150 // extra edge (uninteresting): the client has it, only its parents are new.
151 const haveClosure = new Set();
152 { const stack = [...knownHaves, ...unshallowed]; while (stack.length) { const c = stack.pop(); if (haveClosure.has(c)) continue; haveClosure.add(c); if (!cut.has(c)) for (const p of model.commits.get(c).parents) stack.push(p); } }
154 // 3. interesting commits: walk from wants, never through a had commit (a had want's parents are not visited)
155 const included = new Set();
156 { const stack = wantCommits.filter(w => inDepth.has(w)); while (stack.length) { const c = stack.pop(); if (included.has(c) || haveClosure.has(c) || !inDepth.has(c)) continue; included.add(c); if (!cut.has(c)) for (const p of model.commits.get(c).parents) if (inDepth.has(p)) stack.push(p); } }
158 // 4. uninteresting objects
159 const haveObjects = new Set(haveClosure);
160 if (mode === 'bitmap') {
161 for (const [o] of treeClosure(model, [...haveClosure].map(c => model.commits.get(c).tree))) haveObjects.add(o);
162 } else {
163 // mark_edges_uninteresting: trees of uninteresting parents of interesting commits (not of boundary commits: grafted).
164 // pack-objects passes --objects-edge-aggressive iff --thin: then cmdline haves' trees are marked too.
165 const edges = new Set([...(thinPack ? knownHaves : []), ...unshallowed]);
166 for (const c of included) if (!cut.has(c)) for (const p of model.commits.get(c).parents) if (haveClosure.has(p)) edges.add(p);
167 for (const [o] of treeClosure(model, [...edges].map(c => model.commits.get(c).tree))) haveObjects.add(o);
168 }
169 for (const h of nonCommitHaves) { haveObjects.add(h); if (model.type.get(h) === 'tree') for (const [o] of treeClosure(model, [h])) haveObjects.add(o); }
170 for (const h of haves) if (model.tags.has(h)) haveObjects.add(h);
172 // 5. objects of interesting commits, then filter
173 const f = parseFilter(filter);
174 const info = (o, d) => ({ type: model.type.get(o), depth: d, size: model.size.get(o) });
175 const out = new Set();
176 for (const c of included) out.add(c);
177 for (const [o, d] of treeClosure(model, [...included].map(c => model.commits.get(c).tree))) if (!haveObjects.has(o) && f(info(o, d), model)) out.add(o);
179 // 6. explicitly wanted objects
180 for (const w of wants) {
181 if (!model.type.has(w)) continue;
182 const t = model.type.get(w);
183 if (t === 'commit') continue; // handled by the walk
184 if (t === 'tag') { if (!haveObjects.has(w)) { out.add(w); const c = model.tags.get(w); if (model.commits.has(c) && !haveClosure.has(c) && !included.has(c)) out.add(c); } continue; }
185 // tree/blob: traversal mode drops it only if it is literally a have; bitmap mode drops it if any have reaches it
186 const dropped = mode === 'bitmap' ? haveObjects.has(w) : (haves.includes(w) || [...nonCommitHaves].some(h => h === w || (model.type.get(h) === 'tree' && treeClosure(model, [h]).has(w))));
187 if (dropped) continue;
188 // a user-given want is unfiltered and its tree entries count from depth 0 (tree:N omits depth > N); a want that git has
189 // already flagged NOT_USER_GIVEN is filtered like any tree (see userGiven). With bitmaps every wanted object is a tip that
190 // filter_bitmap_exclude_type keeps whatever the walk flagged (pack-bitmap.c find_tip_objects); its contents are filtered.
191 const given = mode === 'bitmap' || userGiven(model, w, wants, included, filter), off = given ? 1 : 0;
192 if (given || f(info(w, 0), model)) out.add(w);
193 if (t === 'tree' && (given || f(info(w, 0), model))) for (const [o, d] of treeClosure(model, [w])) if (!haveObjects.has(o) && f(info(o, Math.max(0, d - off)), model)) out.add(o);
194 }
195 const shallowLines = [...walk0.boundary].filter(c => !clientShallow.has(c));
196 return { objects: out, shallow: shallowLines.sort(), unshallow: [...unshallowed].sort(), includedCommits: [...included], uninteresting: haveObjects, mode, boundary: walk0.boundary, inDepth };
197}
199// git applies an object filter only to objects flagged NOT_USER_GIVEN (list_objects_filter__filter_object). The commit walk
200// flags every walked commit's root tree before any pending tree is processed, and walking a wanted tree flags everything
201// inside it. So a wanted tree/blob stays "user given" — unfiltered, with its tree's entries counted one level shallower —
202// only if it is neither a walked commit's root tree nor inside an earlier wanted tree (list-objects.c, do_traverse).
203export function userGiven(model, w, wants, walkedCommits, filter) {
204 for (const c of walkedCommits) if (model.commits.get(c).tree === w) return false;
205 // the deepest tree depth whose contents a tree:N filter (or a combine with one) lets git walk into
206 const N = Math.min(...[...String(filter ?? '').matchAll(/tree:(\d+)/g)].map(m => +m[1]), Infinity);
207 const prior = [];
208 for (const e of wants) {
209 if (e === w) break;
210 prior.push(e);
211 if (model.type.get(e) !== 'tree') continue;
212 const eGiven = userGiven(model, e, prior.slice(0, -1), walkedCommits, filter);
213 if (!eGiven && !(0 < N)) continue; // a filtered root at depth 0 is skipped whole under tree:0
214 // walk as git does: a processed tree flags all its entries; a subtree is processed iff its depth < N.
215 // A user-given root does not count a level (its entries are depth 0); a filtered root is depth 0.
216 const stack = [[e, eGiven ? -1 : 0]], seen = new Map();
217 while (stack.length) {
218 const [t, d] = stack.pop();
219 if (seen.has(t) && seen.get(t) <= d) continue; seen.set(t, d);
220 for (const x of model.trees.get(t) ?? []) {
221 if (x.oid === w) return false;
222 if (x.type === 'tree' && d + 1 < N) stack.push([x.oid, d + 1]);
223 }
224 }
225 }
226 return true;
227}
229// ---- the two-sided bound any correct server must satisfy, independent of git's traversal heuristics ----
230// required: reachable(wants) \ reachable(haves) exactly (haves cut at the request's shallow set), after the filter.
231// had: everything reachable from haves (same cut) — any extra the server sends must be in here.
232// bitmap: the server answers from a bitmap index when it can; then a wanted tree/blob no have reaches is required unfiltered.
233export function requiredAndHad(model, { wants, haves = [], deepen, deepenSince, deepenNot = [], filter, shallow = [], bitmap = false }) {
234 const exp = expectedObjects(model, { wants, haves, deepen, deepenSince, deepenNot, filter, shallow, bitmap: true });
235 const bitmapMode = bitmap && exp.mode === 'bitmap';
236 // expectedObjects in bitmap mode == precise set, except deepen forces traversal mode; recompute precisely here
237 const peel = o => (model.tags.has(o) ? model.tags.get(o) : o);
238 const exclude = new Set();
239 for (const r of deepenNot) { const o = model.refs.get(r); if (!o) continue; for (const c of reachableCommits(model, [peel(o)]).commits.keys()) exclude.add(c); }
240 const knownHaves = haves.map(peel).filter(h => model.commits.has(h));
241 const clientShallow = shallow.filter(c => model.commits.has(c));
242 let wantCommits = wants.map(peel).filter(w => model.commits.has(w));
243 const walk0 = reachableCommits(model, wantCommits, { depth: deepen ?? Infinity, since: deepenSince ?? 0, exclude });
244 const deepening = deepen !== undefined || !!deepenSince || deepenNot.length > 0;
245 const unshallowed = deepening ? clientShallow.filter(c => walk0.commits.has(c) && !walk0.boundary.has(c)) : [];
246 wantCommits = [...wantCommits, ...unshallowed.flatMap(u => model.commits.get(u).parents)];
247 const { commits: inDepth } = walk0;
248 const boundary = walk0.boundary;
249 const cut = new Set([...clientShallow, ...boundary]);
250 const had = new Set();
251 // the client has what it declared shallow (and an unshallowed commit is treated as had by the server)
252 { const stack = [...knownHaves, ...unshallowed]; while (stack.length) { const c = stack.pop(); if (had.has(c)) continue; had.add(c); if (!cut.has(c)) for (const p of model.commits.get(c).parents) stack.push(p); } }
253 const hadCommits = new Set(had);
254 for (const [o] of treeClosure(model, [...hadCommits].map(c => model.commits.get(c).tree))) had.add(o);
255 for (const h of haves) { if (!model.type.has(h)) continue; had.add(h); if (model.type.get(h) === 'tree') for (const [o] of treeClosure(model, [h])) had.add(o); if (model.tags.has(h)) had.add(model.tags.get(h)); }
256 const f = parseFilter(filter);
257 const info = (o, d) => ({ type: model.type.get(o), depth: d, size: model.size.get(o) });
258 const required = new Set();
259 // walk from wants, never through a had commit (a had, shallow want has no parents from the client's point of view)
260 const included = [];
261 { const seen = new Set(); const stack = wantCommits.filter(w => inDepth.has(w)); while (stack.length) { const c = stack.pop(); if (seen.has(c) || hadCommits.has(c)) continue; seen.add(c); included.push(c); if (!cut.has(c)) for (const p of model.commits.get(c).parents) if (inDepth.has(p)) stack.push(p); } }
262 for (const c of included) required.add(c);
263 for (const [o, d] of treeClosure(model, included.map(c => model.commits.get(c).tree))) if (!had.has(o) && f(info(o, d), model)) required.add(o);
264 for (const w of wants) {
265 if (!model.type.has(w) || model.commits.has(w) || had.has(w)) continue;
266 const given = model.tags.has(w) || bitmapMode || userGiven(model, w, wants, included, filter), off = given ? 1 : 0;
267 if (given || f(info(w, 0), model)) required.add(w);
268 if (model.tags.has(w)) { const c = model.tags.get(w); if (!hadCommits.has(c)) required.add(c); }
269 if (model.type.get(w) === 'tree' && (given || f(info(w, 0), model))) for (const [o, d] of treeClosure(model, [w])) if (!had.has(o) && f(info(o, Math.max(0, d - off)), model)) required.add(o);
270 }
271 return { required, had, shallow: exp.shallow, unshallow: exp.unshallow, boundary, inDepth };
272}
Diagnostics4 git calls · 233 ms
CallStatusSizeTime0233 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2005.8 KiB227 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=227 git.response.bytes=5926 git.response.cache=MISS git.response.duration_ms=227 git.fetch.wants=f91793637e1f18eecc9cf4aab457b532ef6097c0 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5926 git.pkt.payload.max=5905 git.sideband.band1.bytes=5904 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2cf130b80efad
- Request
- 5cad0831…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format