GitHubivarvong

ply

main
ply6 git calls93 ms33 KiB4 cachedprotocol v2
400 lines32 KiBRaw
1// pushTree: a whole tree as one commit, judged by git. Every push lands on `git http-backend` with receive.fsckObjects;
2// after each one: `git fsck --strict` on the server, the tree oid against git's own index (update-index --cacheinfo +
3// write-tree over the same files), `git ls-tree -r` modes and oids, and a `git clone` whose checkout is compared byte
4// for byte and mode for mode. A repeat push must carry exactly the objects `git rev-list --objects new ^old` names.
5import { test, beforeAll as before, afterAll as after } from 'vitest';
6import assert from 'node:assert/strict';
7import { join } from 'node:path';
8import { mkdirSync, writeFileSync, symlinkSync, chmodSync, readFileSync, lstatSync, readlinkSync, readdirSync } from 'node:fs';
9import { execFileSync } from 'node:child_process';
10import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, commitFiles, fsckProblemsOf, fsckProblems } from './server.js';
11import { startProxy } from './proxy.js';
12import { pushTree, buildTree, fileEntries, selectObjects } from '../src/tree.js';
13import { filesFromDir, pushDir } from '../src/tree-node.js';
14import { encodeTree } from '../src/objects.js';
15import { concat, te } from '../src/bytes.js';
16import { fromHex } from '../src/sha1.js';
17import { withLimits } from '../src/limits.js';
18import { Tracer, arrayExporter, flattenLine } from '../src/otel.js';
19import { unwrap, unwrapAsync } from './unwrap.js';
21let root, cleanup, srv, n = 0;
22before(async () => { ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root); });
23after(async () => { await srv.close(); cleanup(); });
25const A = { name: 'deploy', email: 'deploy@x' };
26const GITLINK = 'c0ffee'.padEnd(40, '0');
27const traced = () => { const ex = arrayExporter(); return { ex, tracer: new Tracer({ exporter: ex }), spans: () => ex.lines.flatMap(flattenLine) }; };
28const emptyRepo = () => { const name = `t${n++}`; const bare = makeBareRepo(root, name); return { name, bare, url: `${srv.url}/${name}.git` }; };
29// counts requests per service; `before(kind)` runs ahead of each one (the CAS tests move the ref there)
30const counting = (hook = () => {}) => {
31 const c = { 'git-upload-pack': 0, 'git-receive-pack': 0 };
32 const f = async (url, init) => { const kind = String(url).split('/').pop(); c[kind] = (c[kind] ?? 0) + 1; await hook(kind, c); return fetch(url, init); };
33 return { c, f };
34};
35const fsck = bare => assert.deepEqual(fsckProblems(bare), []);
37// deterministic incompressible-ish content (an LCG's hex), so pack sizes are stable across runs
38const text = (seed, len) => { let x = (seed + 1) >>> 0, s = ''; while (s.length < len) { x = (Math.imul(x, 1103515245) + 12345) >>> 0; s += x.toString(16).padStart(8, '0'); } return s.slice(0, len) + '\n'; };
39// 200 files over 20 directories, plus the shapes that bite: exec bit, symlink, gitlink, dir-vs-file sort order, unicode
40function site() {
41 const files = {};
42 for (let d = 0; d < 20; d++) for (let f = 0; f < 10; f++) files[`d${String(d).padStart(2, '0')}/f${f}.txt`] = text(d * 100 + f, 1024);
43 Object.assign(files, {
44 'bin/run.sh': { data: '#!/bin/sh\necho hi\n', mode: '100755' },
45 'link': { mode: '120000', data: 'd00/f0.txt' },
46 'vendor/lib': { mode: '160000', oid: GITLINK },
47 'a.b': 'x\n', 'a/x': 'y\n', 'a0': 'z\n', 'a-b/c': 'w\n',
48 'ünï/cödé.txt': 'unicode\n',
49 });
50 return files;
52// path → { mode, oid } for the files, as `git ls-tree -r` reports them
53const expected = files => new Map(unwrap(fileEntries(files)).map(e => [e.path, { mode: e.mode, oid: e.oid }]));
54const lsTree = (bare, rev) => new Map(git(bare, ['-c', 'core.quotePath=false', 'ls-tree', '-r', '-z', '--full-tree', rev]).split('\0').filter(Boolean).map(l => { const [meta, path] = l.split('\t'); const [mode, , oid] = meta.split(' '); return [path, { mode: mode.replace(/^0/, ''), oid }]; }));
55// git's tree for the same files: its own index, built entry by entry, then write-tree
56function gitWriteTree(files) {
57 const dir = join(root, `wt${n++}`); git(root, ['init', '-q', dir]);
58 for (const e of unwrap(fileEntries(files))) {
59 const oid = e.mode === '160000' ? e.oid : gitBuf(dir, ['hash-object', '-w', '--stdin'], Buffer.from(e.data)).toString().trim();
60 git(dir, ['update-index', '--add', '--cacheinfo', `${e.mode},${oid},${e.path}`]);
61 }
62 return git(dir, ['write-tree']).trim();
64// clone and compare the checkout with the files: bytes, the exec bit, symlink targets, nothing extra
65function cloneMatches(bare, files) {
66 const dir = join(root, `c${n++}`); git(root, ['clone', '-q', bare, dir]);
67 const seen = new Set();
68 const walk = rel => { for (const name of readdirSync(join(dir, rel))) { if (name === '.git') continue; const p = rel ? `${rel}/${name}` : name; const st = lstatSync(join(dir, p)); if (st.isDirectory() && !st.isSymbolicLink()) { if (readdirSync(join(dir, p)).length === 0) seen.add(p); else walk(p); } else seen.add(p); } };
69 walk('');
70 for (const e of unwrap(fileEntries(files))) {
71 const full = join(dir, e.path);
72 assert.ok(seen.has(e.path), `${e.path} missing from the checkout`);
73 if (e.mode === '160000') { assert.ok(lstatSync(full).isDirectory(), `${e.path}: a gitlink checks out as an empty directory`); continue; }
74 if (e.mode === '120000') { assert.ok(lstatSync(full).isSymbolicLink(), `${e.path} is a symlink`); assert.equal(readlinkSync(full), Buffer.from(e.data).toString(), e.path); continue; }
75 assert.deepEqual(new Uint8Array(readFileSync(full)), e.data, `${e.path} bytes`);
76 assert.equal(!!(lstatSync(full).mode & 0o100), e.mode === '100755', `${e.path} exec bit`);
77 }
78 assert.equal(seen.size, unwrap(fileEntries(files)).length, `checkout has exactly the files: ${[...seen].filter(p => !(p in files)).join(', ')}`);
80const receivePack = spans => spans.filter(s => s.name === 'git.receive-pack');
82test('first push creates the ref: tree == git write-tree, ls-tree modes/oids, fsck --strict, clone matches', async () => {
83 const r = emptyRepo(), files = site(), t = traced();
84 const res = await unwrapAsync(pushTree(r.url, { ref: 'refs/heads/main', files, author: A, message: 'deploy 1' }, { tracer: t.tracer, verify: true }));
85 assert.equal(res.oldCommit, null); assert.equal(res.attempt, 0); assert.equal(res.noop, false);
86 assert.equal(git(r.bare, ['rev-parse', 'refs/heads/main']).trim(), res.newCommit);
87 assert.equal(git(r.bare, ['rev-list', '--count', 'main']).trim(), '1', 'a root commit');
88 assert.equal(res.newTree, gitWriteTree(files), 'tree oid equals git\'s index for the same files');
89 assert.deepEqual(lsTree(r.bare, 'main'), expected(files));
90 fsck(r.bare); cloneMatches(r.bare, files);
91 const rp = receivePack(t.spans())[0].attrs;
92 // every blob (200 + 6 regular + 1 symlink, none shared) + every tree (20 + bin a a-b vendor ünï + root) + the commit
93 assert.equal(rp['git.push.pack.objects'], 207 + 26 + 1);
94 assert.equal(res.objects.sent, 234); assert.equal(res.objects.treeDeltas, 0);
95 console.log(` push 1 (create): ${rp['git.push.pack.objects']} objects, ${rp['git.push.pack.bytes']} B pack`);
96 // span names follow the written convention (test/pure.test.js), and the root carries its outcome
97 const NAME = /^(git\.(upload-pack|receive-pack)( [a-z-]+)?|git\.[a-z_]+(\.[a-z_]+)?)$/;
98 for (const s of t.spans()) assert.match(s.name, NAME);
99 assert.equal(t.spans().find(s => s.name === 'git.push_tree').attrs['git.push_tree.outcome'], 'ok');
100});
102test('repeat pushes are thin: only changed blobs and the trees on their paths (as REF_DELTAs), == rev-list --objects new ^old', async () => {
103 const r = emptyRepo(), files = site();
104 const first = await unwrapAsync(pushTree(r.url, { files, author: A, message: 'deploy 1' }, { tracer: new Tracer() }));
105 const t1 = traced(); const { c, f } = counting(); const cache = new Map();
106 // change one file, add one, delete one, in three different directories
107 const next = { ...files, 'd07/f3.txt': text(9999, 1024), 'd11/new.txt': 'new\n' }; delete next['d03/f5.txt'];
108 const second = await unwrapAsync(pushTree(r.url, { files: next, author: A, message: 'deploy 2' }, { tracer: t1.tracer, fetchImpl: f, cache }));
109 assert.equal(second.oldCommit, first.newCommit);
110 assert.equal(git(r.bare, ['rev-parse', 'main']).trim(), second.newCommit);
111 const want = git(r.bare, ['rev-list', '--objects', '--no-object-names', second.newCommit, `^${first.newCommit}`]).trim().split('\n');
112 const rp = receivePack(t1.spans())[0].attrs;
113 assert.equal(want.length, 7, 'commit + root, d03, d07, d11 trees + 2 blobs');
114 assert.equal(rp['git.push.pack.objects'], want.length, `pack carries exactly git's new objects (${want.length})`);
115 assert.equal(second.objects.sentBlobs, 2); assert.equal(second.objects.sentTrees, 4); assert.equal(second.objects.treeDeltas, 4, 'every changed tree is a delta against the one it replaces');
116 assert.equal(second.treesRead, 4, 'the parent\'s root and the three changed directories, nothing else');
117 assert.equal(c['git-upload-pack'], 1 + 1 + 4, 'ls-refs, the parent commit, four trees');
118 assert.deepEqual(lsTree(r.bare, 'main'), expected(next)); assert.equal(second.newTree, gitWriteTree(next));
119 fsck(r.bare); cloneMatches(r.bare, next);
121 // push 3, a one-byte change, with the head hint and push 2's cache: no tree is fetched, the commit fetch overlaps ls-refs
122 const third = { ...next, 'd19/f9.txt': next['d19/f9.txt'].replace(/^./, '#') };
123 const t2 = traced(); const warm = counting();
124 const res3 = await unwrapAsync(pushTree(r.url, { files: third, author: A, message: 'deploy 3' }, { tracer: t2.tracer, cache, head: second.newCommit, fetchImpl: warm.f }));
125 assert.equal(res3.hint, 'used'); assert.equal(res3.oldCommit, second.newCommit);
126 assert.equal(res3.treesRead, 0, 'root and d19 come from the cache');
127 assert.equal(warm.c['git-upload-pack'], 2, 'ls-refs and the hinted commit, concurrently');
128 const rp3 = receivePack(t2.spans())[0].attrs;
129 assert.equal(rp3['git.push.pack.objects'], 4, 'commit + root + d19 + one blob');
130 console.log(` push 2 (3 changes, cold): ${rp['git.push.pack.objects']} objects, ${rp['git.push.pack.bytes']} B pack, ${c['git-upload-pack']} upload-pack requests; push 3 (1 byte, hint + cache): ${rp3['git.push.pack.objects']} objects, ${rp3['git.push.pack.bytes']} B pack, ${warm.c['git-upload-pack']} upload-pack requests`);
131 assert.ok(rp['git.push.pack.bytes'] < 2_000, `push 2 pack ${rp['git.push.pack.bytes']} B`);
132 fsck(r.bare); cloneMatches(r.bare, third);
134 // push 4 restores push 1's files. Only the parent's trees are consulted: the three blobs push 1 had and push 3 does not
135 // (d03/f5, d07/f3, d19/f9) are on the server in older history but are sent again. Correct, not minimal.
136 const back = await unwrapAsync(pushTree(r.url, { files, author: A, message: 'revert to 1' }, { tracer: new Tracer() }));
137 assert.equal(back.newTree, first.newTree, 'same files, same tree as push 1');
138 assert.equal(back.objects.sentBlobs, 3); assert.equal(back.objects.sentTrees, 5, 'root, d03, d07, d11, d19');
139 fsck(r.bare);
140});
142test('identical files: noop, no receive-pack; base ≠ ref with identical files still creates the branch', async () => {
143 const r = emptyRepo(), files = { 'a.txt': 'a\n' };
144 const first = await unwrapAsync(pushTree(r.url, { files, author: A, message: 'one' }, { tracer: new Tracer() }));
145 const t = traced();
146 const again = await unwrapAsync(pushTree(r.url, { files, author: A, message: 'two' }, { tracer: t.tracer }));
147 assert.equal(again.noop, true); assert.equal(again.newCommit, first.newCommit);
148 assert.equal(receivePack(t.spans()).length, 0);
149 assert.equal(t.spans().find(s => s.name === 'git.push_tree').attrs['git.push_tree.outcome'], 'noop');
150 const branch = await unwrapAsync(pushTree(r.url, { ref: 'refs/heads/preview', base: 'refs/heads/main', files, author: A, message: 'preview' }, { tracer: new Tracer() }));
151 assert.equal(branch.objects.sent, 1, 'only the commit: the server has the tree');
152 assert.equal(git(r.bare, ['rev-parse', 'preview^']).trim(), first.newCommit);
153 fsck(r.bare);
154 // create is a CAS on "absent": a second create of the same branch is cas_conflict on both attempts
155 await assert.rejects(unwrapAsync(pushTree(r.url, { ref: 'refs/heads/preview', base: 'refs/heads/main', files: { 'b.txt': 'b' }, author: A, message: 'again' }, { tracer: new Tracer() })), e => e.code === 'cas_conflict' && e.attempt === 1);
156 await assert.rejects(unwrapAsync(pushTree(r.url, { ref: 'refs/heads/x', base: 'refs/heads/nope', files, author: A, message: 'm' }, { tracer: new Tracer() })), e => e.code === 'not_found');
157});
159test('cas_conflict: a ref moved between ls-refs and receive-pack is redone once from a fresh head; strictHead is not', async () => {
160 const r = emptyRepo();
161 const first = await unwrapAsync(pushTree(r.url, { files: { 'a.txt': 'one\n' }, author: A, message: 'one' }, { tracer: new Tracer() }));
162 const work = join(root, `w${n++}`); git(root, ['clone', '-q', r.bare, work]);
163 let moved = null;
164 const { f } = counting(kind => { if (kind === 'git-receive-pack' && moved === null) moved = commitFiles(work, r.bare, { 'theirs.txt': 'theirs\n' }, 'competitor'); });
165 const t = traced();
166 const res = await unwrapAsync(pushTree(r.url, { files: { 'a.txt': 'ours\n' }, author: A, message: 'ours' }, { tracer: t.tracer, fetchImpl: f, verify: true }));
167 assert.equal(res.attempt, 1, 'retried once');
168 assert.equal(res.oldCommit, moved, 'the retry\'s parent is the commit that won the race');
169 assert.equal(git(r.bare, ['rev-parse', 'main']).trim(), res.newCommit);
170 assert.deepEqual([...lsTree(r.bare, 'main').keys()], ['a.txt'], 'the tree is exactly `files`; the competitor stays in history');
171 assert.equal(git(r.bare, ['rev-parse', 'main~2']).trim(), first.newCommit);
172 const roots = t.spans().filter(s => s.name === 'git.push_tree');
173 assert.deepEqual(roots.map(s => s.attrs['git.push_tree.outcome']), ['cas_conflict', 'ok']);
174 assert.equal(receivePack(t.spans()).map(s => s.attrs['git.push.ok']).join(), 'false,true');
175 fsck(r.bare);
177 // strictHead: the hint is a precondition. Stale before the push → cas_conflict with no receive-pack; lost race → not retried.
178 const stale = counting();
179 await assert.rejects(unwrapAsync(pushTree(r.url, { files: { 'a.txt': 'x\n' }, author: A, message: 'x' }, { tracer: new Tracer(), head: first.newCommit, strictHead: true, fetchImpl: stale.f })), e => e.code === 'cas_conflict' && e.actual === res.newCommit);
180 assert.equal(stale.c['git-receive-pack'], 0);
181 let moved2 = null;
182 const race = counting(kind => { if (kind === 'git-receive-pack' && moved2 === null) { git(work, ['pull', '-q', '--ff-only', r.bare, 'main']); moved2 = commitFiles(work, r.bare, { 'again.txt': 'x\n' }, 'competitor 2'); } });
183 await assert.rejects(unwrapAsync(pushTree(r.url, { files: { 'a.txt': 'y\n' }, author: A, message: 'y' }, { tracer: new Tracer(), head: res.newCommit, strictHead: true, fetchImpl: race.f })), e => e.code === 'cas_conflict' && e.attempt === 0);
184 assert.equal(race.c['git-receive-pack'], 1);
185 assert.equal(git(r.bare, ['rev-parse', 'main']).trim(), moved2);
186 fsck(r.bare);
187});
189test('after an acknowledged push, a failed read-back is ambiguous and the ref is at our commit', async () => {
190 const r = emptyRepo();
191 const { f } = counting((kind, c) => { if (kind === 'git-upload-pack' && c['git-receive-pack'] > 0) throw new TypeError('fetch failed'); });
192 const err = await unwrapAsync(pushTree(r.url, { files: { 'a.txt': 'a\n' }, author: A, message: 'm' }, { tracer: new Tracer(), fetchImpl: f, verify: true })).catch(e => e);
193 assert.equal(err.code, 'ambiguous'); assert.equal(err.original, 'network');
194 assert.equal(git(r.bare, ['rev-parse', 'main']).trim(), err.expected, 'the push landed');
195 fsck(r.bare);
196});
198test('a pack over maxPackBytes is refused with its measured size before receive-pack is contacted', async () => {
199 const r = emptyRepo();
200 const files = {}; for (let i = 0; i < 100; i++) files[`f${i}.bin`] = text(i, 2048);
201 const t = traced(); const { c, f } = counting();
202 const max = 64 * 1024;
203 const err = await unwrapAsync(pushTree(r.url, { files, author: A, message: 'big' }, { tracer: t.tracer, fetchImpl: f, limits: withLimits({ maxPackBytes: max }) })).catch(e => e);
204 assert.equal(err.code, 'limit_exceeded'); assert.equal(err.what, 'push pack bytes'); assert.equal(err.max, max);
205 assert.ok(err.got > max, `measured ${err.got}`);
206 assert.equal(c['git-receive-pack'], 0, 'nothing uploaded');
207 assert.equal(receivePack(t.spans())[0].attrs['git.push.pack.bytes'], err.got, 'the span records the size that was refused');
208 assert.equal(git(r.bare, ['for-each-ref']).trim(), '', 'ref not created');
209 // the same files under the default cap land
210 await unwrapAsync(pushTree(r.url, { files, author: A, message: 'big' }, { tracer: new Tracer() })); fsck(r.bare);
211});
213test('duplicate names: encodeTree refuses them (git fsck duplicateEntries); pushTree refuses before any I/O; case-only twins are git-valid', async () => {
214 const oid = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391';
215 assert.throws(() => unwrap(encodeTree([{ mode: '100644', name: 'a', oid }, { mode: '100644', name: 'a', oid }])), e => e.code === 'bad_request' && e.entry === 'a');
216 assert.throws(() => unwrap(encodeTree([{ mode: '100644', name: 'a', oid }, { mode: '100644', name: 'a.b', oid }, { mode: '40000', name: 'a', oid: '4b825dc642cb6eb9a060e54bf8d69288fbee4904' }])), e => e.code === 'bad_request' && e.entry === 'a', 'file a and dir a are not neighbours in tree order');
217 // git's verdict on the bytes encodeTree no longer produces: a file `a` twice, and a file `a` beside a directory `a`
218 const dir = join(root, `dup${n++}`); git(root, ['init', '-q', '--bare', dir]);
219 const entry = (mode, name, o) => concat([te.encode(`${mode} ${name}\0`), fromHex(o)]);
220 gitBuf(dir, ['hash-object', '-w', '--stdin'], Buffer.alloc(0));
221 gitBuf(dir, ['hash-object', '-w', '-t', 'tree', '--stdin'], Buffer.alloc(0));
222 for (const bytes of [concat([entry('100644', 'a', oid), entry('100644', 'a', oid)]), concat([entry('100644', 'a', oid), entry('100644', 'a.b', oid), entry('40000', 'a', '4b825dc642cb6eb9a060e54bf8d69288fbee4904')])]) {
223 const tree = gitBuf(dir, ['hash-object', '--literally', '-w', '-t', 'tree', '--stdin'], Buffer.from(bytes)).toString().trim();
224 const out = fsckProblemsOf(dir, tree);
225 assert.ok(out.some(l => /^error in tree [0-9a-f]{40}: duplicateEntries: /.test(l)), `git fsck rejects it: ${out.join(' | ')}`);
226 }
227 // pushTree: the same mistakes in `files` are bad_request before a request is made
228 const { c, f } = counting();
229 const r = emptyRepo();
230 const bad = [
231 { 'a': 'x', 'a/b': 'y' },
232 [{ path: 'a/b', data: 'x' }, { path: 'a', data: 'y' }],
233 [{ path: 'p', data: 'x' }, { path: 'p', data: 'y' }],
234 { 'x': { mode: '100600', data: 'x' } },
235 { 'x': { mode: '160000', oid: 'nope' } },
236 { 'x': { mode: '120000', data: '' } },
237 { '.git/config': 'x' }, { 'a//b': 'x' }, { 'a/../b': 'x' },
238 ];
239 for (const files of bad) await assert.rejects(unwrapAsync(pushTree(r.url, { files, author: A, message: 'm' }, { tracer: new Tracer(), fetchImpl: f })), e => e.code === 'bad_request', JSON.stringify(files));
240 assert.deepEqual(c, { 'git-upload-pack': 0, 'git-receive-pack': 0 });
241 // README and readme are two names to git: fsck --strict accepts the tree; they collide only on a case-insensitive checkout
242 const res = await unwrapAsync(pushTree(r.url, { files: { README: 'upper\n', readme: 'lower\n' }, author: A, message: 'case' }, { tracer: new Tracer() }));
243 fsck(r.bare);
244 assert.deepEqual([...lsTree(r.bare, res.newCommit).keys()].sort(), ['README', 'readme']);
245});
247test('the pure steps: buildTree equals git for edge orders; selectObjects skips what the server has', () => {
248 const files = { 'a/x': '1', 'a.b': '2', 'a0': '3', 'a-b/c': '4', 'b': { mode: '120000', data: 'a/x' } };
249 assert.equal(unwrap(buildTree(unwrap(fileEntries(files)))).root, gitWriteTree(files));
250 assert.equal(unwrap(buildTree([])).root, '4b825dc642cb6eb9a060e54bf8d69288fbee4904', 'no files is git\'s empty tree');
251 assert.equal(unwrap(buildTree(unwrap(fileEntries(new Map([['m', 'x']]))))).root, unwrap(buildTree(unwrap(fileEntries({ m: 'x' })))).root, 'a Map is the object form');
252 const built = unwrap(buildTree(unwrap(fileEntries({ 'x/1': 'same', 'y/1': 'same', 'z': 'other' }))));
253 const all = selectObjects(built);
254 assert.equal(all.blobs.length, 2, 'one blob at two paths is sent once'); assert.equal(all.trees.length, 2, 'x and y are one tree: root + that tree');
255 const none = selectObjects(built, { have: new Set([built.root]) });
256 assert.equal(none.trees.length + none.blobs.length, 0);
257 assert.throws(() => unwrap(fileEntries({ a: 'x' }, withLimits({ maxObjectSize: 0 }))), e => e.code === 'limit_exceeded' && e.what === 'object size');
258 assert.throws(() => unwrap(fileEntries({ a: 'x', b: 'y' }, withLimits({ maxPackObjects: 1 }))), e => e.code === 'limit_exceeded' && e.what === 'files');
259 assert.throws(() => unwrap(buildTree(unwrap(fileEntries({ a: 'x', b: 'y' })), withLimits({ maxTreeEntries: 1 }))), e => e.code === 'limit_exceeded' && e.what === 'tree entries');
260 assert.throws(() => unwrap(fileEntries(null)), e => e.code === 'bad_request');
261});
263test('Artifacts-like host (blob:none only): the repeat push reads the parent\'s trees from one wide fetch', async () => {
264 const r = emptyRepo(), files = site();
265 const filterOf = body => /filter ([^\n]+)\n/.exec(Buffer.from(body).toString('latin1'))?.[1] ?? null;
266 const proxy = await startProxy(srv.url, ctx => (ctx.stage === 'fetch' && filterOf(ctx.reqBody) && filterOf(ctx.reqBody) !== 'blob:none' ? { type: 'status', status: 400, contentType: 'text/plain; charset=utf-8', body: 'Unsupported Git fetch filter; only blob:none is supported.' } : { type: 'pass' }));
267 try {
268 const url = `${proxy.url}/${r.name}.git`;
269 await unwrapAsync(pushTree(url, { files, author: A, message: 'one' }, { tracer: new Tracer() }));
270 const next = { ...files, 'd07/f3.txt': 'changed\n' };
271 const t = traced(), quirks = {};
272 const res = await unwrapAsync(pushTree(url, { files: next, author: A, message: 'two' }, { tracer: t.tracer, quirks }));
273 assert.equal(quirks.treeFilter, 'blob:none');
274 const fetches = t.spans().filter(s => s.name === 'git.upload-pack fetch');
275 assert.equal(fetches.filter(s => s.attrs['git.fetch.filter.fallback'] === 'blob:none').length, 1, 'one blob:none fetch');
276 assert.equal(res.treesRead, 2, 'root and d07, both answered from the harvest');
277 assert.equal(receivePack(t.spans())[0].attrs['git.push.pack.objects'], 4);
278 fsck(r.bare); cloneMatches(r.bare, next);
279 } finally { await proxy.close(); }
280});
282test('filesFromDir/pushDir (Node): exec bit, symlinks as 120000, nested dirs, .git and empty dirs skipped, filter; FIFO refused', async () => {
283 const src = join(root, `src${n++}`);
284 const put = (p, data, mode) => { mkdirSync(join(src, p, '..'), { recursive: true }); writeFileSync(join(src, p), data); if (mode) chmodSync(join(src, p), mode); };
285 put('index.html', '<h1>hi</h1>\n'); put('assets/app.js', 'console.log(1)\n'); put('bin/build', '#!/bin/sh\n', 0o755);
286 put('deep/a/b/c/d.txt', 'deep\n'); put('.git/config', 'not ours\n'); put('node_modules/x/index.js', 'skip me\n');
287 put('owner-noexec', 'x\n', 0o644); put('group-exec-only', 'x\n', 0o654);
288 mkdirSync(join(src, 'empty'));
289 symlinkSync('assets/app.js', join(src, 'app-link')); symlinkSync('../index.html', join(src, 'assets/up'));
290 symlinkSync('does/not/exist', join(src, 'dangling'));
291 const filter = path => !path.startsWith('node_modules');
292 const files = await unwrapAsync(filesFromDir(src, { filter }));
293 const byPath = new Map(files.map(e => [e.path, e]));
294 assert.deepEqual([...byPath.keys()].sort(), ['app-link', 'assets/app.js', 'assets/up', 'bin/build', 'dangling', 'deep/a/b/c/d.txt', 'group-exec-only', 'index.html', 'owner-noexec']);
295 assert.equal(byPath.get('bin/build').mode, '100755'); assert.equal(byPath.get('group-exec-only').mode, '100644', 'git looks at the owner\'s execute bit only');
296 assert.equal(byPath.get('app-link').mode, '120000'); assert.equal(Buffer.from(byPath.get('dangling').data).toString(), 'does/not/exist');
297 const r = emptyRepo();
298 const res = await unwrapAsync(pushDir(r.url, { dir: src, filter, author: A, message: 'dir' }, { tracer: new Tracer() }));
299 assert.equal(res.newTree, gitWriteTree(files));
300 // git's own reading of the same directory: `git add -A` in a copy (minus node_modules) writes the same tree
301 const wt = join(root, `addA${n++}`); execFileSync('cp', ['-R', src, wt]); execFileSync('rm', ['-rf', join(wt, '.git'), join(wt, 'node_modules')]);
302 git(wt, ['init', '-q']); git(wt, ['add', '-A']);
303 assert.equal(git(wt, ['write-tree']).trim(), res.newTree, 'same tree as git add -A');
304 fsck(r.bare); cloneMatches(r.bare, Object.fromEntries(files.map(e => [e.path, { mode: e.mode, data: e.data }])));
305 execFileSync('mkfifo', [join(src, 'pipe')]);
306 await assert.rejects(unwrapAsync(filesFromDir(src)), e => e.code === 'bad_request' && e.path === 'pipe');
307 await assert.rejects(unwrapAsync(filesFromDir(src, { limits: withLimits({ maxObjectSize: 8 }), filter: p => p !== 'pipe' })), e => e.code === 'limit_exceeded' && e.what === 'object size');
308});
310// macOS only (skipped by name elsewhere: Linux keeps a name's bytes, and git there does too). A name created decomposed
311// (NFD) stays decomposed on APFS; `git add` under the configuration `git init` writes there (core.precomposeunicode =
312// true, compat/precompose_utf8.c) records it precomposed. The oracle is git on a copy of the same directory with no
313// global or system configuration: its ls-files names and its write-tree. Nothing expected comes from ply.
314// README "Your own code failing": a pushDir / filesFromDir filter that throws is bad_request, what: 'filter', the thrown
315// value as cause; the walk precedes any request, so it is definite and git's ref does not move.
316test('filesFromDir/pushDir: a filter that throws is bad_request naming filter, cause the thrown value, no request, ref unmoved', async () => {
317 const r = emptyRepo();
318 const work = join(root, `${r.name}-w`); mkdirSync(work); git(work, ['init', '-q', '-b', 'main']);
319 commitFiles(work, r.bare, { 'a.txt': 'a\n' });
320 const before = git(r.bare, ['rev-parse', 'refs/heads/main']).trim();
321 const src = join(root, `${r.name}-src`); mkdirSync(join(src, 'sub'), { recursive: true });
322 writeFileSync(join(src, 'keep.txt'), 'k\n'); writeFileSync(join(src, 'sub', 'x.txt'), 'x\n');
323 // throws on the second path it is shown (sub, after keep.txt in sorted order), so it is not the first call alone
324 const thrown = new Error('filter boom'); let calls = 0;
325 const filter = () => { calls++; if (calls === 2) throw thrown; return true; };
326 const walked = await filesFromDir(src, { filter });
327 assert.equal(walked.ok, false);
328 assert.deepEqual([walked.error.code, walked.error.what, walked.error.cause === thrown, walked.error.message], ['bad_request', 'filter', true, 'caller-supplied filter threw: filter boom']);
329 calls = 0;
330 const { c, f } = counting();
331 const pushed = await pushDir(r.url, { dir: src, filter, author: A, message: 'dir' }, { fetchImpl: f });
332 assert.equal(pushed.ok, false);
333 assert.deepEqual([pushed.error.code, pushed.error.what, pushed.error.cause === thrown], ['bad_request', 'filter', true]);
334 assert.deepEqual(c, { 'git-upload-pack': 0, 'git-receive-pack': 0 }, 'no request');
335 assert.equal(git(r.bare, ['rev-parse', 'refs/heads/main']).trim(), before, 'git: the ref did not move');
336});
338test.skipIf(process.platform !== 'darwin')('filesFromDir/pushDir on macOS [darwin only]: a decomposed (NFD) name is recorded as git add records it (precomposed)', async () => {
339 const src = join(root, `nfd${n++}`);
340 // Latin e + combining acute, a decomposed directory holding a decomposed file, Hangul jamo, kana + combining dakuten
341 const names = ['é.txt', 'Å/ö.txt', '한.txt', 'が.txt', 'plain.txt'];
342 for (const name of names) { mkdirSync(join(src, name, '..'), { recursive: true }); writeFileSync(join(src, name), `${name}\n`); }
343 assert.ok(readdirSync(src).includes('é.txt'), 'the file system kept the decomposed name');
344 const wt = join(root, `nfdgit${n++}`);
345 execFileSync('cp', ['-R', src, wt]);
346 const plain = { env: { PATH: process.env.PATH, HOME: wt, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' } };
347 git(wt, ['init', '-q'], plain);
348 assert.equal(git(wt, ['config', 'core.precomposeunicode'], plain).trim(), 'true', 'git init\'s own default on this file system');
349 git(wt, ['add', '-A'], plain);
350 const gitNames = git(wt, ['ls-files', '-z'], plain).split('\0').filter(Boolean).sort();
351 assert.ok(gitNames.includes('é.txt'), 'git recorded e-acute precomposed');
352 const files = await unwrapAsync(filesFromDir(src));
353 assert.deepEqual(files.map(f => f.path).sort(), gitNames);
354 const r = emptyRepo();
355 const res = await unwrapAsync(pushDir(r.url, { dir: src, author: A, message: 'nfd' }, { tracer: new Tracer() }));
356 assert.equal(res.newTree, git(wt, ['write-tree'], plain).trim(), 'same tree as git add -A');
357 fsck(r.bare);
358});
360// macOS only (skipped by name elsewhere). git's precompose is not Unicode NFC: compat/precompose_utf8.c passes each
361// directory entry through iconv UTF-8-MAC → UTF-8, which joins decomposed pairs and nothing else. NFC also rewrites
362// characters standing alone (KELVIN SIGN → K, OHM SIGN → Ω, U+F900 → U+8C48, U+0958 → U+0915 U+093C) and reorders marks;
363// git does neither, keeps a name holding a code point beyond the BMP as read, and does not join the Balinese pairs.
364// The oracle is git on a copy of the same directory with no global or system configuration (git add -A, ls-files -z):
365// the re-review's 13 names, then a sweep of every code point NFC rewrites, the decomposed form of every precomposed
366// character (Hangul sampled), marks in and out of canonical order, and names with an emoji. Nothing expected is ply's.
367test.skipIf(process.platform !== 'darwin')('filesFromDir on macOS [darwin only]: a name NFC would rewrite is recorded as git add records it, which is not NFC', async () => {
368 const src = join(root, `precompose${n++}`); mkdirSync(src);
369 const named = ['K-kelvin.txt', 'Ω-ohm.txt', 'Å-angstrom.txt', '豈-cjkcompat.txt', 'क़-qa.txt', 'क़-qa-decomposed.txt',
370 'ײַ-yiddish.txt', 'יִ-hiriq.txt', 'ẛ̣.txt', 'é.txt', '̈́-diaeresis-tonos.txt', '⫝̸-forking.txt', 'plain.txt',
371 '\u{1F600}é.txt', 'é\u{1F600}.txt', 'ệ-out-of-order.txt', 'ệ-in-order.txt', 'à.txt', 'ᬆ-balinese.txt'];
372 const bmp = []; const astral = [];
373 for (let cp = 0x80; cp <= 0x10FFFF; cp++) {
374 if ((cp >= 0xD800 && cp <= 0xDFFF) || (cp >= 0xAC00 && cp <= 0xD7A3 && cp % 97 !== 0)) continue;
375 const c = String.fromCodePoint(cp); const nfd = c.normalize('NFD'); const into = cp > 0xFFFF ? astral : bmp;
376 if (c.normalize('NFC') !== c) into.push(c); // NFC rewrites it standing alone
377 if (nfd !== c) into.push(nfd); // a precomposed character, decomposed
378 }
379 const marks = ['̀', '́', '̂', '̈', '̊', '̣', '̧', '̛', '̓', 'ͅ', '̀', '̈́'];
380 for (const b of ['a', 'o', 'α', 'Å']) for (const m1 of marks) for (const m2 of marks) bmp.push(b + m1 + m2);
381 // 12 samples to a name, `_` between them (nothing joins `_`); a name is at most 255 bytes
382 const packed = (tag, list) => Array.from({ length: Math.ceil(list.length / 12) }, (_, i) => `${tag}${i}_${list.slice(i * 12, i * 12 + 12).join('_')}`);
383 const names = [...named, ...packed('b', bmp), ...packed('s', astral)];
384 for (const name of names) writeFileSync(join(src, name), '');
385 mkdirSync(join(src, 'é\u{1F600}')); writeFileSync(join(src, 'é\u{1F600}', 'é.txt'), ''); // each path component by itself
386 assert.equal(readdirSync(src).length, names.length + 1, 'the file system kept every name');
387 const wt = join(root, `precomposegit${n++}`);
388 execFileSync('cp', ['-R', src, wt]);
389 const plain = { env: { PATH: process.env.PATH, HOME: wt, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' } };
390 git(wt, ['init', '-q'], plain);
391 assert.equal(git(wt, ['config', 'core.precomposeunicode'], plain).trim(), 'true', 'git init\'s own default on this file system');
392 git(wt, ['add', '-A'], plain);
393 const gitNames = git(wt, ['ls-files', '-z'], plain).split('\0').filter(Boolean).sort();
394 assert.equal(gitNames.length, names.length + 1);
395 // the sample is one where git and NFC part: git kept these as created, and joined the plain pair
396 for (const kept of ['K-kelvin.txt', 'क़-qa.txt', '豈-cjkcompat.txt', '\u{1F600}é.txt', 'é\u{1F600}/é.txt']) assert.ok(gitNames.includes(kept), `git recorded ${kept}`);
397 assert.ok(gitNames.includes('é.txt') && !gitNames.includes('K-kelvin.txt'), 'git joins a pair and leaves a singleton');
398 const files = await unwrapAsync(filesFromDir(src));
399 assert.deepEqual(files.map(f => f.path).sort(), gitNames);
400});
Diagnostics6 git calls · 93 ms
CallStatusSizeTime093 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=80
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch20010 KiB82 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=10270
git.response.cache=MISS
git.response.duration_ms=82
git.fetch.wants=bee7c09bc514c2819229fb99963f4d3827a2f903
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=5
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=10270
git.pkt.payload.max=8192
git.sideband.band1.bytes=10238
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0f9e98ab5332c
Request
6d35ab3f…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format