Artifactspublic-v0

ply

main
ply4 git calls218 ms47 KiB2 cachedprotocol v2
412 lines31 KiBRaw
1// checkoutTree / checkoutDir: a ref out of a remote into a directory, judged by `git clone` of the same ref. Every
2// fixture lives in a bare repo behind `git http-backend`; the clone and ply's checkout must be `diff -r` identical, and
3// a walk of both compares every entry's type, permission bits, bytes and symlink target. Round trips are counted at
4// the fetch boundary and compared with the per-file baseline (readCommit → listTree per directory → readFile per blob,
5// the first examples/checkout-tree.mjs, before checkoutDir). Also: a blob:none-only host (Artifacts), a host that refuses many wants, lying servers
6// (wrong blob, unwanted object, wrong commit, duplicate tree entry), `path`, the limits, and the writer's path rules.
7import { test, beforeAll as before, afterAll as after } from 'vitest';
8import assert from 'node:assert/strict';
9import { join } from 'node:path';
10import { mkdirSync, symlinkSync, readdirSync, lstatSync, readlinkSync, readFileSync, existsSync, writeFileSync } from 'node:fs';
11import { execFileSync } from 'node:child_process';
12import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, commitFiles, initRepo } from './server.js';
13import { startProxy } from './proxy.js';
14import { checkoutTree, treeFiles, canonMode } from '../src/checkout.js';
15import { CHECKOUT_BATCH, checkoutDir, writeFiles } from '../src/checkout-node.js';
16import { readCommit, listTree, readFile } from '../src/read.js';
17import { DEFAULT_LIMITS, withLimits } from '../src/limits.js';
18import { Tracer, arrayExporter, flattenLine } from '../src/otel.js';
19import { errorKind, ok } from '../src/result.js';
20import { unwrap, unwrapAsync } from './unwrap.js';
22let root, cleanup, srv, R, n = 0;
23const traced = () => { const ex = arrayExporter(); return { tracer: new Tracer({ exporter: ex }), spans: () => ex.lines.flatMap(flattenLine) }; };
24// Success unwraps. An Err is a test bug on a path the oracle says git accepts.
25const must = r => { if (!r.ok) throw r.error; return r.value; };
26// writeFiles and checkoutDir's own refusals have no span (checkoutTree owns git.checkout). The caller stamps error.kind.
27const stamped = e => {
28 const ex = arrayExporter();
29 new Tracer({ exporter: ex }).start('git.checkout.write').error(e, e.code).end();
30 const flat = flattenLine(ex.lines[0])[0];
31 assert.equal(flat.attrs['error.kind'], errorKind(e));
32 assert.equal(flat.attrs['error.type'], e.code);
33 return e;
34};
35const refusal = async (p) => { const r = await p; assert.equal(r.ok, false); return stamped(r.error); };
36// counts HTTP requests by service; `rewrite(body)` may replace an upload-pack request body (the lying servers)
37const counting = (rewrite = null) => {
38 const c = { requests: 0, bodies: [] };
39 const f = async (url, init) => {
40 c.requests++;
41 let body = init?.body instanceof Uint8Array ? Buffer.from(init.body).toString('latin1') : null;
42 if (body !== null) c.bodies.push(body);
43 if (body !== null && rewrite) { const b = rewrite(body); if (b !== body) { body = b; return fetch(url, { ...init, body: Buffer.from(b, 'latin1') }); } }
44 return fetch(url, init);
45 };
46 return { c, f };
47};
48const fresh = () => join(root, `o${n++}`);
49const WRITE = { limits: DEFAULT_LIMITS, platform: process.platform };
50const req = (dir, over = {}) => ({ dir, ref: null, path: '', ...over });
51const copts = (tracer, over = {}) => ({
52 auth: null, tracer, parent: null, fetchImpl: null, limits: DEFAULT_LIMITS, signal: null,
53 quirks: null, transport: null, batch: CHECKOUT_BATCH, platform: process.platform, ...over,
54});
55const wants = body => (body.match(/^[0-9a-f]{4}want [0-9a-f]{40}$/gm) ?? []).length;
57// the fixture: nested dirs, exec bits, symlinks (file, directory, dangling, upward), a gitlink, unicode and spaces,
58// an empty file, one blob at two paths, a 3 MiB file, sort-order neighbours (a.b, a/, a0, a-b)
59before(async () => {
60 ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root);
61 const bare = makeBareRepo(root, 'r'); const work = join(root, 'wr'); mkdirSync(work); initRepo(work);
62 const big = Buffer.alloc(3 << 20); for (let i = 0; i < big.length; i += 7) big[i] = (i * 2654435761) >>> 24;
63 const files = {
64 'README.md': '# fixture\n', 'empty.txt': '', 'big.bin': big,
65 'src/lib/deep/x/y/z.js': 'export const z = 1;\n', 'src/lib/util.js': 'export {};\n', 'src/main.js': 'import "./lib/util.js";\n',
66 'bin/run.sh': { mode: '100755', data: '#!/bin/sh\necho hi\n' }, 'bin/tool': { mode: '100755', data: '#!/bin/sh\nexit 0\n' },
67 'dir with space/ünï cödé.txt': 'unicode\n', '日本/ファイル.txt': 'nihongo\n', 'emoji/🙂.txt': 'smile\n',
68 'copy/one.txt': 'same bytes\n', 'copy/two.txt': 'same bytes\n',
69 'a.b': '1\n', 'a/x': '2\n', 'a0': '3\n', 'a-b/c': '4\n',
70 };
71 commitFiles(work, bare, files, 'one');
72 symlinkSync('README.md', join(work, 'link-file')); symlinkSync('src/lib', join(work, 'link-dir'));
73 symlinkSync('does/not/exist', join(work, 'link-dangling')); symlinkSync('../../README.md', join(work, 'src/lib/up'));
74 commitFiles(work, bare, {}, 'links');
75 git(work, ['update-index', '--add', '--cacheinfo', `160000,${git(work, ['rev-parse', 'HEAD~1']).trim()},vendor/sub`]);
76 git(work, ['-c', 'user.name=seed', '-c', 'user.email=seed@x', 'commit', '-q', '-m', 'gitlink']); git(work, ['push', '-q', bare, 'HEAD:refs/heads/main']);
77 R = { bare, work, url: `${srv.url}/r.git`, files: Object.keys(files).length + 4 + 1 };
78});
79after(async () => { await srv.close(); cleanup(); });
81const clone = (ref = null) => { const dir = fresh(); git(root, ['clone', '-q', ...(ref ? ['-b', ref] : []), R.bare, dir]); return dir; };
82// diff -r without following links, then a walk: type, permission bits, bytes, link targets, empty directories
83function sameTree(want, got) {
84 try { execFileSync('diff', ['-r', '--no-dereference', '-x', '.git', want, got], { stdio: 'pipe' }); } catch (e) { assert.fail(`diff -r: ${e.stdout}${e.stderr}`); }
85 const walk = (base, rel = '', out = new Map()) => {
86 for (const name of readdirSync(join(base, rel)).sort()) {
87 if (name === '.git' && rel === '') continue;
88 const p = rel ? `${rel}/${name}` : name, st = lstatSync(join(base, p));
89 if (st.isSymbolicLink()) out.set(p, `link ${readlinkSync(join(base, p))}`);
90 else if (st.isDirectory()) { out.set(p, `dir ${(st.mode & 0o777).toString(8)}`); walk(base, p, out); }
91 else out.set(p, `file ${(st.mode & 0o777).toString(8)} ${readFileSync(join(base, p)).toString('base64')}`);
92 }
93 return out;
94 };
95 const a = walk(want), b = walk(got);
96 assert.deepEqual([...b.keys()], [...a.keys()], 'same paths');
97 for (const [p, v] of a) assert.equal(b.get(p), v, p);
98 return a.size;
101test('checkoutDir == git clone: diff -r and modes; 3 round trips against the per-file baseline', async () => {
102 const t = traced(), { c, f } = counting(), dir = fresh();
103 const r = must(await checkoutDir(R.url, req(dir), copts(t.tracer, { fetchImpl: f })));
104 const entries = sameTree(clone(), dir);
105 assert.equal(r.commit, git(R.bare, ['rev-parse', 'main']).trim()); assert.equal(r.tree, git(R.bare, ['rev-parse', 'main^{tree}']).trim());
106 assert.equal(r.ref, 'refs/heads/main');
107 assert.equal(r.entries.length, R.files);
108 assert.deepEqual(r.written, { files: R.files - 5, bytes: r.written.bytes, symlinks: 4, gitlinks: 1 });
109 assert.ok(lstatSync(join(dir, 'vendor/sub')).isDirectory() && readdirSync(join(dir, 'vendor/sub')).length === 0, 'a gitlink is an empty directory');
110 assert.equal(lstatSync(join(dir, 'bin/run.sh')).mode & 0o100, 0o100);
111 // ls-refs, the blob:none tree pack, one wants-list for every blob
112 assert.equal(c.requests, 3, `${c.requests} requests`);
113 const fetches = t.spans().filter(s => s.name === 'git.upload-pack fetch');
114 assert.deepEqual(fetches.map(s => s.attrs['git.fetch.filter']), ['blob:none', '']);
115 const unique = new Set(r.entries.filter(e => e.mode !== '160000').map(e => e.oid)).size;
116 assert.equal(fetches[1].attrs['git.fetch.wants.count'], unique, 'every unique blob in one request');
117 const blobs = t.spans().find(s => s.name === 'git.checkout.blobs').attrs;
118 assert.equal(blobs['git.checkout.blobs.outcome'], 'ok'); assert.equal(blobs['git.checkout.blobs.fetches'], 1);
119 assert.equal(blobs['git.checkout.blobs.buffered'], R.files - 1); assert.equal(blobs['git.checkout.blobs.streamed'], 0);
120 assert.equal(t.spans().find(s => s.name === 'git.checkout').attrs['git.checkout.outcome'], 'ok');
121 // round trips as the example counts them: spans that sent a request body
122 const trips = t.spans().filter(s => s.attrs['git.request.bytes'] != null).length;
123 assert.equal(trips, 3);
125 // the baseline (what the first examples/checkout-tree.mjs did): readCommit once, listTree per directory, readFile per blob, by commit oid
126 const base = counting(), opts = { tracer: new Tracer(), fetchImpl: base.f, cache: new Map(), quirks: {} };
127 const { oid } = await unwrapAsync(readCommit(R.url, { ref: 'main' }, opts));
128 let read = 0;
129 const walk = async path => {
130 for (const e of (await unwrapAsync(listTree(R.url, { ref: oid, path }, opts))).entries) {
131 const p = path ? `${path}/${e.name}` : e.name;
132 if (e.type === 'tree') await walk(p); else if (e.type === 'blob') { await new Response((await unwrapAsync(readFile(R.url, { ref: oid, path: p }, opts))).stream).arrayBuffer(); read++; }
133 }
134 };
135 await walk('');
136 assert.equal(read, R.files - 1);
137 assert.ok(base.c.requests >= 2 * read, `baseline ${base.c.requests} requests for ${read} blobs`);
138 console.log(` checkout of ${R.files} entries (${entries} paths on disk, ${unique} unique blobs): checkoutTree ${c.requests} requests; per-file baseline ${base.c.requests} requests`);
139});
141test('the iterator: entries and files agree, sizes before bytes, a gitlink has no stream; path = a subtree', async () => {
142 const r = await unwrapAsync(checkoutTree(R.url, { ref: 'main' }, { tracer: new Tracer() }));
143 const ls = new Map(git(R.bare, ['-c', 'core.quotePath=false', 'ls-tree', '-r', '-z', '-l', 'main']).split('\0').filter(Boolean).map(l => { const [meta, path] = l.split('\t'); const [mode, , oid, size] = meta.trim().split(/\s+/); return [path, { mode, oid, size: size === '-' ? 0 : Number(size) }]; }));
144 assert.deepEqual(r.entries.map(e => [e.path, e.mode, e.oid]), [...ls].map(([p, e]) => [p, e.mode, e.oid]), 'entries == git ls-tree -r, in its order');
145 const seen = new Set();
146 for await (const f of r.files) {
147 const want = ls.get(f.path);
148 assert.equal(f.mode, want.mode, f.path); assert.equal(f.oid, want.oid); assert.equal(f.size, want.size, f.path);
149 if (f.mode === '160000') { assert.equal(f.stream, null); seen.add(f.path); continue; }
150 const bytes = Buffer.from(await new Response(f.stream).arrayBuffer());
151 assert.ok(bytes.equals(gitBuf(R.bare, ['cat-file', 'blob', f.oid])), f.path);
152 seen.add(f.path);
153 }
154 assert.equal(seen.size, ls.size, 'every entry once');
156 // path: a subtree, entries relative to it; the same bytes as the clone's subdirectory
157 const dir = fresh();
158 const sub = must(await checkoutDir(R.url, req(dir, { path: 'src/lib' }), copts(new Tracer())));
159 assert.equal(sub.tree, git(R.bare, ['rev-parse', 'main:src/lib']).trim()); assert.equal(sub.path, 'src/lib');
160 assert.deepEqual(sub.entries.map(e => e.path), ['deep/x/y/z.js', 'up', 'util.js']);
161 sameTree(join(clone(), 'src/lib'), dir);
162 for (const [path, re] of [['nope', /not found/], ['README.md', /not a directory/], ['src/nope/x', /not found/]]) await assert.rejects(unwrapAsync(checkoutTree(R.url, { path }, { tracer: new Tracer() })), e => e.code === 'not_found' && re.test(e.message), path);
163 await assert.rejects(unwrapAsync(checkoutTree(R.url, { path: '../x' }, { tracer: new Tracer() })), e => e.code === 'bad_request');
164 // an older commit by oid (no ls-refs), and a ref that is not there
165 const old = git(R.bare, ['rev-parse', 'main~2']).trim(), counted = counting();
166 const o = await unwrapAsync(checkoutTree(R.url, { ref: old }, { tracer: new Tracer(), fetchImpl: counted.f }));
167 assert.equal(o.entries.length, R.files - 5, 'no links, no gitlink yet'); assert.equal(counted.c.requests, 1, 'an oid needs no ls-refs');
168 await assert.rejects(unwrapAsync(checkoutTree(R.url, { ref: 'nope' }, { tracer: new Tracer() })), e => e.code === 'not_found');
169 await assert.rejects(unwrapAsync(checkoutTree(R.url, { ref: git(R.bare, ['rev-parse', 'main:README.md']).trim() }, { tracer: new Tracer() })), e => e.code === 'not_found');
170});
172test('Artifacts-like host (blob:none only): the same checkout in the same 3 round trips, no refused request', async () => {
173 const filterOf = body => /filter ([^\n]+)\n/.exec(Buffer.from(body).toString('latin1'))?.[1] ?? null;
174 const proxy = await startProxy(srv.url, ctx => (ctx.stage === 'fetch' && filterOf(ctx.reqBody) && filterOf(ctx.reqBody) !== 'blob:none' ? { type: 'status', status: 400, contentType: 'text/plain; charset=utf-8', body: 'Unsupported Git fetch filter; only blob:none is supported.' } : { type: 'pass' }));
175 try {
176 const { c, f } = counting(), dir = fresh();
177 must(await checkoutDir(`${proxy.url}/r.git`, req(dir), copts(new Tracer(), { fetchImpl: f })));
178 sameTree(clone(), dir);
179 assert.equal(c.requests, 3);
180 assert.equal(proxy.log.filter(l => l.spec === 'status').length, 0, 'nothing refused: blob:none is the only filter asked for');
181 } finally { await proxy.close(); }
182});
184test('a host that refuses many wants: one refusal, then a blob per request, remembered on quirks', async () => {
185 const proxy = await startProxy(srv.url, ctx => (ctx.stage === 'fetch' && wants(ctx.reqBody.toString('latin1')) > 1 ? { type: 'status', status: 400, contentType: 'text/plain', body: 'one want per request' } : { type: 'pass' }));
186 try {
187 const { c, f } = counting(), dir = fresh(), quirks = {}, t = traced();
188 const r = must(await checkoutDir(`${proxy.url}/r.git`, req(dir), copts(t.tracer, { fetchImpl: f, quirks })));
189 sameTree(clone(), dir);
190 assert.equal(quirks.blobWants, 'single');
191 const blobPaths = r.entries.filter(e => e.mode !== '160000').length;
192 assert.equal(c.requests, 2 + 1 + blobPaths, 'ls-refs, trees, the refused batch, then one per path');
193 const s = t.spans().find(x => x.name === 'git.checkout.blobs').attrs;
194 assert.equal(s['git.checkout.blobs.refused'], 'http'); assert.equal(s['git.checkout.blobs.streamed'], blobPaths);
195 // with the quirk known, no batch is tried
196 const again = counting();
197 await unwrapAsync(checkoutTree(`${proxy.url}/r.git`, { ref: r.commit }, { tracer: new Tracer(), fetchImpl: again.f, quirks })).then(async x => { for await (const g of x.files) await g.stream?.cancel(); });
198 assert.equal(again.c.bodies.filter(b => wants(b) > 1).length, 0);
199 } finally { await proxy.close(); }
200});
202test('lying servers fail closed with pack_corrupt: wrong blob in a batch, wrong blob alone (file removed), wrong commit, duplicate entry', async () => {
203 const readme = git(R.bare, ['rev-parse', 'main:README.md']).trim(), other = git(R.bare, ['rev-parse', 'main:a.b']).trim();
204 // 1. a batch: the server answers README.md's want with a blob nobody asked for → pack_corrupt before any file is yielded
205 const decoy = gitBuf(R.bare, ['hash-object', '-w', '--stdin'], Buffer.from('not in any tree\n')).toString().trim();
206 const liar1 = counting(body => (!body.includes('filter ') && body.includes(`want ${readme}\n`) ? body.replace(`want ${readme}\n`, `want ${decoy}\n`) : body));
207 const r1 = await unwrapAsync(checkoutTree(R.url, {}, { tracer: new Tracer(), fetchImpl: liar1.f }));
208 const yielded = [];
209 const err1 = await (async () => { for await (const f of r1.files) if (f.stream) yielded.push(f.path); })().catch(e => e);
210 assert.equal(err1.code, 'pack_corrupt'); assert.match(err1.message, /served but not wanted/);
211 assert.deepEqual(yielded, [], 'no blob was yielded from the lying batch');
213 // 2. a blob fetched on its own (batch: 1): the stream errors, checkoutDir removes the file and returns Err
214 const liar2 = counting(body => (!body.includes('filter ') && body.includes(`want ${readme}\n`) ? body.replace(`want ${readme}\n`, `want ${other}\n`) : body));
215 const dir = fresh();
216 const refused2 = await checkoutDir(R.url, req(dir), copts(new Tracer(), { fetchImpl: liar2.f, batch: 1 }));
217 assert.equal(refused2.ok, false);
218 const err2 = stamped(refused2.error);
219 assert.equal(err2.code, 'pack_corrupt'); assert.match(err2.message, /hashes to/);
220 assert.ok(!existsSync(join(dir, 'README.md')), 'the unverified file is removed');
222 // 3. the tree fetch answered for another commit: the wanted commit is not in the pack
223 const head = git(R.bare, ['rev-parse', 'main']).trim(), older = git(R.bare, ['rev-parse', 'main~1']).trim();
224 const liar3 = counting(body => body.replace(`want ${head}\n`, `want ${older}\n`));
225 await assert.rejects(unwrapAsync(checkoutTree(R.url, { ref: head }, { tracer: new Tracer(), fetchImpl: liar3.f })), e => e.code === 'pack_corrupt' && /not in the blob:none pack/.test(e.message));
226});
228test('a tree naming one entry twice (a symlink `a` beside a directory `a`) is refused before anything is written', async () => {
229 const bare = makeBareRepo(root, 'dup');
230 const hash = (type, bytes) => gitBuf(bare, ['hash-object', '--literally', '-w', '-t', type, '--stdin'], bytes).toString().trim();
231 const entry = (mode, name, oid) => Buffer.concat([Buffer.from(`${mode} ${name}\0`), Buffer.from(oid, 'hex')]);
232 const blob = hash('blob', Buffer.from('pwned\n')), link = hash('blob', Buffer.from('/tmp'));
233 const inner = hash('tree', entry('100644', 'x', blob));
234 const tree = hash('tree', Buffer.concat([entry('120000', 'a', link), entry('40000', 'a', inner)]));
235 const commit = git(bare, ['-c', 'user.name=x', '-c', 'user.email=x@x', 'commit-tree', tree, '-m', 'dup']).trim();
236 git(bare, ['update-ref', 'refs/heads/main', commit]);
237 const dir = fresh();
238 const dup = await checkoutDir(`${srv.url}/dup.git`, req(dir), copts(new Tracer()));
239 assert.equal(dup.ok, false);
240 const dupErr = stamped(dup.error);
241 assert.equal(dupErr.code, 'pack_corrupt');
242 assert.match(dupErr.message, /duplicate entry "a"/);
243 assert.deepEqual(readdirSync(dir), []);
244});
246test('limits: entries walked, depth, the tree pack, blob batches split under maxPackBytes, a single blob over it refused', async () => {
247 // the tree pack is 15 objects (commit + 14 trees); walking it visits 36 entries (22 files + 13 trees + 1 gitlink)
248 await assert.rejects(unwrapAsync(checkoutTree(R.url, {}, { tracer: new Tracer(), limits: withLimits({ maxPackObjects: 20 }), batch: 5 })), e => e.code === 'limit_exceeded' && e.what === 'checkout entries' && e.got === 21);
249 await assert.rejects(unwrapAsync(checkoutTree(R.url, {}, { tracer: new Tracer(), limits: withLimits({ maxPathDepth: 4 }) })), e => e.code === 'limit_exceeded' && e.what === 'path depth');
250 await assert.rejects(unwrapAsync(checkoutTree(R.url, {}, { tracer: new Tracer(), limits: withLimits({ maxPackBytes: 512 }) })), e => e.code === 'limit_exceeded' && e.what === 'pack bytes');
251 for (const batch of [0, 1.5, 'x', 200_001]) await assert.rejects(unwrapAsync(checkoutTree(R.url, {}, { tracer: new Tracer(), batch })), e => e.code === 'bad_request' && e.what === 'batch');
252 // a cap the tree pack fits under but the whole blob batch does not, by one byte: split in half, both halves refetched
253 // under the cap and buffered (every fetch, streamed or buffered, stays under maxPackBytes)
254 const t0 = traced();
255 must(await checkoutDir(R.url, req(fresh()), copts(t0.tracer)));
256 const whole = t0.spans().find(s => s.name === 'git.upload-pack fetch' && s.attrs['git.fetch.filter'] === '').attrs['git.pack.bytes'];
257 const max = whole - 1, t = traced(), dir = fresh();
258 must(await checkoutDir(R.url, req(dir), copts(t.tracer, { limits: withLimits({ maxPackBytes: max }) })));
259 sameTree(clone(), dir);
260 const blobs = t.spans().find(s => s.name === 'git.checkout.blobs').attrs;
261 assert.equal(blobs['git.checkout.blobs.splits'], 1, 'the batch was split once'); assert.equal(blobs['git.checkout.blobs.fetches'], 2); assert.equal(blobs['git.checkout.blobs.streamed'], 0);
262 for (const s of t.spans().filter(x => x.name === 'git.upload-pack fetch' && x.attrs['git.pack.bytes'] != null)) assert.ok(s.attrs['git.pack.bytes'] <= max, `a buffered pack of ${s.attrs['git.pack.bytes']} B`);
263 // a cap under big.bin's own pack (3 MiB, 58 KiB deflated): the batch splits down to big.bin alone, whose stream is then
264 // over the cap too — limit_exceeded, never a stream read past maxPackBytes (before the endless-body fix it streamed)
265 const t2 = traced(), dir2 = fresh();
266 const capped = await checkoutDir(R.url, req(dir2), copts(t2.tracer, { limits: withLimits({ maxPackBytes: 16 * 1024 }) }));
267 assert.equal(capped.ok, false);
268 const err = stamped(capped.error);
269 assert.equal(err.code, 'limit_exceeded', err.message); assert.equal(err.what, 'pack bytes'); assert.equal(err.max, 16 * 1024);
270 const b2 = t2.spans().find(s => s.name === 'git.checkout.blobs').attrs;
271 // "fetched alone" is read from openBlob's own span, not `streamed`: that counter moves only once openBlob returns, and
272 // whether the cap trips while it reads the object header (one network chunk past 16 KiB) or later in the pump depends
273 // on chunk sizes (CI: the former). Either way failHere ends a git.read.blob span as limit_exceeded.
274 assert.ok(b2['git.checkout.blobs.splits'] >= 1, 'the batch was split');
275 assert.ok(t2.spans().some(s => s.name === 'git.read.blob' && s.attrs['git.read.blob.outcome'] === 'limit_exceeded'), 'big.bin was fetched alone');
276 assert.ok(!existsSync(join(dir2, 'big.bin')), 'the refused blob left no file');
277 // a single blob under the cap still streams: batch 1 with the cap just above big.bin's pack
278 const t3 = traced(), dir3 = fresh();
279 must(await checkoutDir(R.url, req(dir3), copts(t3.tracer, { limits: withLimits({ maxPackBytes: 64 * 1024 }), batch: 1 })));
280 sameTree(clone(), dir3);
281 const read = t3.spans().find(s => s.name === 'git.read.blob' && s.attrs['git.read.blob.size'] === 3 << 20);
282 assert.equal(read.attrs['git.read.blob.verified'], true);
283});
285test('the pure steps: canonMode is git\'s canon_mode; treeFiles bounds a tree reused at many paths', () => {
286 assert.equal(canonMode('100644'), '100644'); assert.equal(canonMode('100664'), '100644'); assert.equal(canonMode('100755'), '100755'); assert.equal(canonMode('100775'), '100755');
287 assert.equal(canonMode('120000'), '120000'); assert.equal(canonMode('160000'), '160000'); assert.equal(canonMode('40000'), '40000');
288 for (const m of ['140000', '100', '1006440', '60000']) assert.equal(canonMode(m), '160000', m); // git ls-tree: every unknown type is a gitlink
289 for (const m of ['0100644', '10x644', '']) assert.equal(canonMode(m), null, m);
290 const oid = i => String(i).padStart(40, '0');
291 // tree k names tree k+1 twice: 2^20 paths from 21 objects
292 const trees = new Map(Array.from({ length: 21 }, (_, k) => [oid(k), k === 20 ? [{ name: 'f', mode: '100644', oid: oid(99) }] : [{ name: 'l', mode: '40000', oid: oid(k + 1) }, { name: 'r', mode: '40000', oid: oid(k + 1) }]]));
293 assert.throws(() => unwrap(treeFiles(o => ok(trees.get(o)), oid(0), { limits: withLimits({ maxPackObjects: 1000 }) })), e => e.code === 'limit_exceeded' && e.what === 'checkout entries' && e.got === 1001);
294 assert.throws(() => unwrap(treeFiles(() => ok([{ name: 'x', mode: '0100644', oid: oid(1) }]), oid(0))), e => e.code === 'pack_corrupt' && /cannot check out/.test(e.message));
295 assert.deepEqual(unwrap(treeFiles(() => ok([{ name: 'x', mode: '140000', oid: oid(1) }]), oid(0))), [{ path: 'x', mode: '160000', oid: oid(1) }]);
296 assert.deepEqual(unwrap(treeFiles(o => ok(trees.get(o) ?? []), oid(19))).map(e => e.path), ['l/f', 'r/f']);
297});
299test('writeFiles refuses traversal, .git, absolute paths, a non-empty dir, and writing through a symlink it made', async () => {
300 const s = text => new ReadableStream({ start(c) { c.enqueue(new TextEncoder().encode(text)); c.close(); } });
301 const one = (...files) => (async function* () { yield* files; })();
302 const outside = fresh(); mkdirSync(outside);
303 for (const path of ['../escape', '/abs', 'a/../../escape', '.git/config', '.GIT/config', 'a//b', 'git~1/x']) {
304 const dir = fresh();
305 const e = await refusal(writeFiles(dir, one({ path, mode: '100644', stream: s('x') }), WRITE));
306 assert.equal(e.code, 'bad_request', path);
307 assert.deepEqual(readdirSync(dir), [], path);
308 }
309 // a symlink `a` → outside, then a file under `a/`: refused, nothing lands outside
310 const dir = fresh();
311 const through = await refusal(writeFiles(dir, one({ path: 'a', mode: '120000', stream: s(outside) }, { path: 'a/pwned', mode: '100644', stream: s('x') }), WRITE));
312 assert.equal(through.kind, 'collision');
313 assert.equal(through.code, 'bad_request');
314 assert.equal(through.what, 'checkout path');
315 assert.equal(through.path, 'a');
316 assert.deepEqual(readdirSync(outside), []);
317 // the same name twice: the second does not overwrite, and a symlink is not replaced
318 const dir2 = fresh();
319 const twice = await refusal(writeFiles(dir2, one({ path: 'f', mode: '100644', stream: s('one') }, { path: 'f', mode: '100644', stream: s('two') }), WRITE));
320 assert.equal(twice.kind, 'collision');
321 assert.equal(twice.path, 'f');
322 assert.equal(readFileSync(join(dir2, 'f'), 'utf8'), 'one');
323 const dir3 = fresh();
324 const link = await refusal(writeFiles(dir3, one({ path: 'l', mode: '120000', stream: s(join(outside, 'target')) }, { path: 'l', mode: '100644', stream: s('x') }), WRITE));
325 assert.equal(link.kind, 'collision');
326 assert.equal(link.path, 'l');
327 assert.ok(!existsSync(join(outside, 'target')), 'not written through the link');
328 // a symlink target over maxObjectSize is limit_exceeded before the link is made
329 const cap = fresh();
330 const huge = new ReadableStream({ start(c) { c.enqueue(new Uint8Array(8)); c.close(); } });
331 const over = await refusal(writeFiles(cap, one({ path: 'l', mode: '120000', stream: huge }), { ...WRITE, limits: withLimits({ maxObjectSize: 4 }) }));
332 assert.equal(over.kind, 'symlink_target');
333 assert.equal(over.code, 'limit_exceeded');
334 assert.equal(over.what, 'symlink target');
335 assert.deepEqual(readdirSync(cap), []);
336 // a non-empty target dir is refused before any request
337 const full = fresh(); mkdirSync(full); writeFileSync(join(full, 'keep'), 'x');
338 const { c, f } = counting();
339 const occupied = await refusal(checkoutDir(R.url, req(full), copts(new Tracer(), { fetchImpl: f })));
340 assert.equal(occupied.kind, 'not_empty');
341 assert.equal(occupied.code, 'bad_request');
342 assert.equal(occupied.what, 'dir');
343 assert.equal(c.requests, 0);
344 const blank = await refusal(checkoutDir('http://unused.example/r.git', req(''), copts(new Tracer())));
345 assert.equal(blank.kind, 'dir');
346 assert.equal(blank.code, 'bad_request');
347 assert.equal(blank.what, 'dir');
348});
350// Hole #4 (space/fsck-tree): modes git only warns about are checked out as git checks them out (canon_mode: 100664 and
351// 100600 are 100644, 100700 is 100755, 7100644 is 100644, 40755 is a directory, a type git does not know is a gitlink),
352// and a tree git refuses to check out (a .gitmodules symlink) is refused before anything is written.
353test('odd modes check out as git clone checks them out; a .gitmodules symlink is refused as git refuses it', async () => {
354 const lit = (bare, type, data) => gitBuf(bare, ['hash-object', '-t', type, '-w', '--stdin', '--literally'], data).toString().trim();
355 const entry = (mode, name, oid) => Buffer.concat([Buffer.from(`${mode} ${name}\0`), Buffer.from(oid, 'hex')]);
356 const commit = (bare, tree) => { const c = lit(bare, 'commit', Buffer.from(`tree ${tree}\nauthor A <a@b.c> 1700000000 +0000\ncommitter A <a@b.c> 1700000000 +0000\n\nodd\n`)); git(bare, ['update-ref', 'refs/heads/main', c]); return c; };
357 const bare = makeBareRepo(root, 'odd');
358 const blob = lit(bare, 'blob', Buffer.from('#!/bin/sh\necho odd\n'));
359 const sub = lit(bare, 'tree', entry('100644', 'f', blob));
360 const prev = commit(bare, sub);
361 // tree order: a directory sorts as name + '/'
362 const tree = lit(bare, 'tree', Buffer.concat([
363 entry('100600', 'a600', blob), entry('100664', 'a664', blob), entry('100700', 'a700', blob), entry('7100644', 'a7100644', blob),
364 entry('40755', 'd755', sub), entry('1', 'g1', prev), entry('140000', 'g140000', prev),
365 ]));
366 commit(bare, tree);
367 const want = fresh(); git(root, ['-c', 'transfer.fsckObjects=false', 'clone', '-q', bare, want]);
368 const got = fresh();
369 const r = must(await checkoutDir(`${srv.url}/odd.git`, req(got), copts(new Tracer())));
370 assert.deepEqual(r.entries.map(e => `${e.mode} ${e.path}`), ['100644 a600', '100644 a664', '100755 a700', '100644 a7100644', '100644 d755/f', '160000 g1', '160000 g140000']);
371 assert.equal(sameTree(want, got), 8);
372 // .gitmodules as a symlink: fsck --strict gitmodulesSymlink, and git's checkout refuses the path (verify_path)
373 const bad = makeBareRepo(root, 'gitmodules-link');
374 const target = lit(bad, 'blob', Buffer.from('/etc/passwd'));
375 commit(bad, lit(bad, 'tree', Buffer.concat([entry('120000', '.gitmodules', target), entry('100644', 'a', target)])));
376 const gitDir = fresh();
377 assert.throws(() => git(root, ['-c', 'transfer.fsckObjects=false', 'clone', '-q', bad, gitDir], { stdio: 'pipe' }), /invalid path '\.gitmodules'/);
378 const plyDir = fresh();
379 const gitmodules = await refusal(checkoutDir(`${srv.url}/gitmodules-link.git`, req(plyDir), copts(new Tracer())));
380 assert.equal(gitmodules.code, 'pack_corrupt');
381 assert.match(gitmodules.message, /\.gitmodules is a symbolic link/);
382 assert.deepEqual(readdirSync(plyDir), []);
383});
385test('writeFiles on Windows refuses what git for Windows refuses: a backslash (`..\\x` leaves the directory), a stream, a device name', async () => {
386 const s = text => new ReadableStream({ start(c) { c.enqueue(new TextEncoder().encode(text)); c.close(); } });
387 const one = (...files) => (async function* () { yield* files; })();
388 for (const path of ['..\\escape', 'a\\.git\\hooks\\post-checkout', 'x/.git:stream', 'con', 'd/NUL.txt', 'trailing.', 'a<b']) {
389 const dir = fresh();
390 const e = await refusal(writeFiles(dir, one({ path, mode: '100644', stream: s('x') }), { ...WRITE, platform: 'win32' }));
391 assert.ok((e.kind === 'path_refused' && e.code === 'unsupported' && e.what === 'checkout path') || (e.code === 'bad_request' && e.what === 'path'), path);
392 assert.deepEqual(readdirSync(dir), [], path);
393 }
394 // on Linux a backslash is a name byte, as git writes it
395 const dir = fresh();
396 must(await writeFiles(dir, one({ path: 'a\\b', mode: '100644', stream: s('x') }), { ...WRITE, platform: 'linux' }));
397 assert.deepEqual(readdirSync(dir), ['a\\b']);
398});
400test('a file where the checkout directory should be is a filesystem refusal, as git clone refuses that path', async () => {
401 const file = join(root, `notdir${n++}`);
402 writeFileSync(file, 'keep');
403 assert.throws(() => git(root, ['clone', '-q', R.bare, file]), e => e.status !== 0);
404 const { c, f } = counting();
405 const e = await refusal(checkoutDir(R.url, req(file), copts(new Tracer(), { fetchImpl: f })));
406 assert.equal(e.kind, 'filesystem');
407 assert.equal(e.code, 'unsupported');
408 assert.equal(e.what, 'checkout path');
409 assert.equal(e.path, file);
410 assert.equal(c.requests, 0);
411 assert.equal(readFileSync(file, 'utf8'), 'keep');
412});
Diagnostics4 git calls · 218 ms
CallStatusSizeTime0218 ms
discover200152 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit7 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=7
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=7
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=38434
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=38434
git.pkt.payload.max=38339
git.sideband.band1.bytes=38338
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=63
git.pack.bytes=38338
git.pack.ofs_delta=0
git.pack.ref_delta=8
git.pack.chain.max=8
git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77
git.pack.full=55
Open pack map
fetch2009.4 KiB209 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=209
git.response.bytes=9616
git.response.cache=MISS
git.response.duration_ms=209
git.fetch.wants=cd2336466cdce339ff4acc9dfb7cec36d1b86154
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=9616
git.pkt.payload.max=9595
git.sideband.band1.bytes=9594
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2cf8a9eb7efad
Request
222d7b4a…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format