// A stock `git http-backend` repository: uploadpack.allowFilter unset, git's default. Its `fetch=` line advertises no // `filter`, and its upload-pack dies on a `filter` argument with an empty response (no ERR, no flush). Every operation // runs against such a repository and against its twin with allowFilter on (makeBareRepo's fixture), both seeded with // the same commits: the answers must be equal (oids, trees, file bytes) and every write fsck --strict clean. Also: the // learning (one advertisement probe per quirks record, the span attributes that say so, an advertisement in hand // answering first), the pack limits on the wider unfiltered pack (limit_exceeded, never protocol), and an empty response // from a server that does advertise `filter` staying the protocol error it is. import { test, beforeAll as before, afterAll as after } from 'vitest'; import assert from 'node:assert/strict'; import { join } from 'node:path'; import { mkdirSync, writeFileSync, chmodSync } from 'node:fs'; import { createHash } from 'node:crypto'; import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, fsckProblems, initRepo } from './server.js'; import { startProxy } from './proxy.js'; import { editFile } from '../src/edit.js'; import { commitChanges } from '../src/changes.js'; import { pushTree } from '../src/tree.js'; import { readFile, listTree, readCommit } from '../src/read.js'; import { readLog } from '../src/log.js'; import { checkoutTree } from '../src/checkout.js'; import { mergeBranch } from '../src/merge.js'; import { rebaseBranch, cherryPick, revertCommit } from '../src/rebase.js'; import { discover } from '../src/discover.js'; import { createTransport } from '../src/http.js'; import { forgetQuirks } from '../src/transport.js'; import { DEFAULT_LIMITS } from '../src/limits.js'; import { Tracer, arrayExporter, flattenLine } from '../src/otel.js'; import { unwrap } from './unwrap.js'; const T = 1_700_000_000; const A = { name: 'Ada', email: 'ada@x', tz: '+0000', time: T }; let root, cleanup, srv, work, seeded, n = 0; // 200 KB that no zlib level shrinks: the unfiltered snapshot is bigger than any filtered answer by this much const noise = len => { const out = Buffer.alloc(len); for (let i = 0, b = Buffer.alloc(0); i < len; i += 32) { b = createHash('sha256').update(b).update(String(i)).digest(); b.copy(out, i); } return out; }; before(async () => { ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root); work = join(root, 'work'); mkdirSync(work); initRepo(work); const put = files => { for (const [p, v] of Object.entries(files)) { mkdirSync(join(work, p, '..'), { recursive: true }); writeFileSync(join(work, p), v); } }; const commit = (msg, i) => { git(work, ['add', '-A']); git(work, ['commit', '-q', '-m', msg], { env: { ...process.env, GIT_AUTHOR_DATE: `${T - 100 + i} +0000`, GIT_COMMITTER_DATE: `${T - 100 + i} +0000` } }); return git(work, ['rev-parse', 'HEAD']).trim(); }; put({ 'README.md': '# readme\n', 'a/b/c.txt': 'deep MARK\n', 'x.txt': 'x1\nx2\nx3\n', 'y.txt': 'y1\n', 'd/z.txt': 'z\n', 'big.bin': noise(200_000), 'run.sh': '#!/bin/sh\necho hi\n' }); chmodSync(join(work, 'run.sh'), 0o755); const base = commit('base', 0); git(work, ['tag', '-a', '-m', 'v1', 'v1']); git(work, ['checkout', '-q', '-b', 'feat']); put({ 'x.txt': 'x1\nx2 feat\nx3\n' }); const feat = commit('feat', 1); git(work, ['checkout', '-q', '-b', 'feat2', base]); put({ 'd/z.txt': 'z feat2\n' }); const feat2 = commit('feat2', 2); git(work, ['checkout', '-q', 'main']); put({ 'y.txt': 'y1 main\n' }); const main = commit('main', 3); seeded = { base, feat, feat2, main }; }); after(async () => { await srv.close(); cleanup(); }); // Twin repositories with the same refs: `filter` (allowFilter on) and `stock` (git's defaults). function twins() { const name = `t${n++}`; const out = {}; for (const kind of ['filter', 'stock']) { const bare = makeBareRepo(root, `${name}-${kind}`, { stock: kind === 'stock' }); git(work, ['push', '-q', bare, 'refs/heads/*:refs/heads/*', 'refs/tags/*:refs/tags/*']); out[kind] = { bare, url: `${srv.url}/${name}-${kind}.git` }; } return out; } const drain = async stream => { const parts = []; for await (const c of stream) parts.push(Buffer.from(c)); return Buffer.concat(parts); }; const spansOf = ex => ex.lines.flatMap(flattenLine); const traced = () => { const ex = arrayExporter(); return { ex, tracer: new Tracer({ exporter: ex }) }; }; // A result as compared across the twins: streams read to bytes, the file iterator drained, push sizes left out (the thin // pack may pick other bases when the unfiltered fetch harvested more trees; what landed is compared by oid instead). async function plain(v) { if (v instanceof ReadableStream) return (await drain(v)).toString('base64'); if (v instanceof Uint8Array) return Buffer.from(v).toString('base64'); if (v && typeof v[Symbol.asyncIterator] === 'function') { const files = []; for await (const f of v) files.push({ ...f, stream: f.stream === null ? null : (await drain(f.stream)).toString('base64') }); return files; } if (Array.isArray(v)) return Promise.all(v.map(plain)); if (v && typeof v === 'object') { const o = {}; for (const [k, x] of Object.entries(v)) if (k !== 'pushBytes') o[k] = await plain(x); return o; } return v; } test('the fixture: the stock twin advertises no filter on fetch=, the other does (what the bug report saw)', async () => { const { stock, filter } = twins(); const info = async url => unwrap(await discover(unwrap(createTransport(url, { tracer: new Tracer() })))); assert.equal((await info(stock.url)).filter, false); assert.equal((await info(filter.url)).filter, true); }); test('reads: readFile, listTree, readCommit, readLog, checkoutTree answer the same on both; one probe, then no filter', async () => { const { stock, filter } = twins(); const reads = async (url, quirks, tracer) => { const o = () => ({ tracer, quirks }); const out = {}; for (const path of ['README.md', 'a/b/c.txt', 'big.bin', 'run.sh']) out[`readFile ${path}`] = await plain(unwrap(await readFile(url, { path }, o()))); out['readFile v1:x.txt'] = await plain(unwrap(await readFile(url, { ref: 'v1', path: 'x.txt' }, o()))); for (const path of ['', 'a', 'a/b', 'd']) out[`listTree ${path}`] = await plain(unwrap(await listTree(url, { ref: 'feat2', path }, o()))); out.readCommit = await plain(unwrap(await readCommit(url, { ref: 'main' }, o()))); out['readCommit oid'] = await plain(unwrap(await readCommit(url, { ref: seeded.feat }, o()))); out.readLog = await plain(unwrap(await readLog(url, { ref: 'main', limit: 3 }, o()))); out.checkoutTree = await plain(unwrap(await checkoutTree(url, { ref: 'main' }, o()))); out['checkoutTree a'] = await plain(unwrap(await checkoutTree(url, { ref: 'feat', path: 'a' }, o()))); return out; }; const fq = {}, sq = {}; const want = await reads(filter.url, fq, new Tracer()); const t = traced(); const got = await reads(stock.url, sq, t.tracer); assert.deepEqual(got, want); assert.equal(got['readFile big.bin'].stream, gitBuf(stock.bare, ['show', 'main:big.bin']).toString('base64')); assert.equal(fq.treeFilter, 'tree:0'); assert.equal(sq.treeFilter, 'none'); const spans = spansOf(t.ex); const probes = spans.filter(s => s.name === 'git.upload-pack discover'); assert.equal(probes.length, 1, 'one advertisement read for the whole sequence (one quirks record)'); assert.equal(probes[0].attrs['git.server.filter'], false); assert.ok(!probes[0].attrs['git.server.fetch'].includes('filter')); const fetches = spans.filter(s => s.name === 'git.upload-pack fetch'); const filtered = fetches.filter(s => s.attrs['git.fetch.filter'] !== ''); assert.equal(filtered.length, 1, 'the one filtered request is the one the server died on'); assert.equal(filtered[0].attrs['error.type'], 'protocol'); assert.equal(filtered[0].attrs['error.kind'], 'empty_response'); const learned = fetches.filter(s => s.attrs['git.fetch.filter.learned'] === true); assert.equal(learned.length, 1); assert.equal(learned[0].attrs['git.fetch.filter.fallback'], 'none'); assert.equal(learned[0].attrs['git.fetch.filter.requested'], 'tree:0'); for (const s of fetches.filter(x => 'git.fetch.filter.fallback' in x.attrs)) assert.equal(s.attrs['git.fetch.filter.fallback'], 'none'); // checkoutTree's blobs came with its unfiltered trees pack: no blob fetch of its own for (const b of spans.filter(s => s.name === 'git.checkout.blobs')) assert.equal(b.attrs['git.checkout.blobs.fetches'], 0, JSON.stringify(b.attrs)); }); test('writes: editFile, commitChanges, pushTree, mergeBranch, rebaseBranch, cherryPick, revertCommit land the same commits; fsck --strict clean', async () => { const { stock, filter } = twins(); const writes = async (url, quirks) => { const o = () => ({ tracer: new Tracer(), quirks }); const out = {}; out.editFile = await plain(unwrap(await editFile(url, { path: 'a/b/c.txt', find: 'MARK', replace: 'EDITED', author: A, message: 'edit', time: T }, { ...o(), verify: true }))); out.commitChanges = await plain(unwrap(await commitChanges(url, { ref: 'refs/heads/main', changes: [{ kind: 'create', path: 'd/new.txt', content: 'new\n' }, { kind: 'rename', from: 'README.md', to: 'd/README.md' }], author: A, message: 'changes', time: T }, o()))); out.pushTree = await plain(unwrap(await pushTree(url, { ref: 'refs/heads/deploy', base: 'refs/heads/main', files: { 'index.html': '

hi

\n', 'a/b/c.txt': 'site\n' }, author: A, message: 'deploy', time: T }, o()))); out.pushTreeAgain = await plain(unwrap(await pushTree(url, { ref: 'refs/heads/deploy', files: { 'index.html': '

hi

\n', 'a/b/c.txt': 'site 2\n' }, author: A, message: 'deploy 2', time: T }, o()))); out.mergeBranch = await plain(unwrap(await mergeBranch(url, { ref: 'refs/heads/main', from: 'refs/heads/feat', author: A, message: 'merge feat', time: T }, o()))); out.rebaseBranch = await plain(unwrap(await rebaseBranch(url, { ref: 'refs/heads/feat2', onto: 'refs/heads/main', committer: A, time: T }, o()))); out.cherryPick = await plain(unwrap(await cherryPick(url, { ref: 'refs/heads/main', commit: seeded.feat2, committer: A, time: T }, o()))); out.revertCommit = await plain(unwrap(await revertCommit(url, { ref: 'refs/heads/main', commit: out.cherryPick.oid, committer: A, time: T }, o()))); return out; }; const fq = {}, sq = {}; const want = await writes(filter.url, fq); const got = await writes(stock.url, sq); assert.deepEqual(got, want); assert.equal(sq.treeFilter, 'none'); assert.equal(want.mergeBranch.outcome, 'merge'); assert.equal(want.rebaseBranch.outcome, 'rebase'); for (const ref of ['main', 'feat', 'feat2', 'deploy']) assert.equal(git(stock.bare, ['rev-parse', ref]).trim(), git(filter.bare, ['rev-parse', ref]).trim(), ref); assert.equal(gitBuf(stock.bare, ['show', 'main:a/b/c.txt']).toString(), 'deep EDITED\n'); assert.equal(want.mergeBranch.tree, git(stock.bare, ['rev-parse', `${want.mergeBranch.oid}^{tree}`]).trim()); assert.deepEqual(fsckProblems(stock.bare), []); assert.deepEqual(fsckProblems(filter.bare), []); }); test('an advertisement in hand answers first: discover() sets the quirk, the reads that follow send no filter', async () => { const { stock, filter } = twins(); const t = traced(); const tr = unwrap(createTransport(stock.url, { tracer: t.tracer })); const info = unwrap(await discover(tr)); assert.equal(info.filter, false); assert.equal(tr.quirks.treeFilter, 'none'); const quirks = tr.quirks; const rd = unwrap(await readFile(stock.url, { ref: 'main', path: 'a/b/c.txt' }, { tracer: t.tracer, quirks })); assert.equal((await drain(rd.stream)).toString(), 'deep MARK\n'); const fetches = spansOf(t.ex).filter(s => s.name === 'git.upload-pack fetch'); assert.ok(fetches.length > 0 && fetches.every(s => s.attrs['git.fetch.filter'] === ''), 'no filtered request at all'); assert.ok(fetches.some(s => s.attrs['git.fetch.filter.fallback'] === 'none' && s.attrs['git.fetch.filter.learned'] === false)); assert.equal(spansOf(t.ex).filter(s => s.name === 'git.upload-pack discover').length, 1, 'no second probe'); // a host that advertises filter undoes a stale 'none'; a learned blob:none is left alone (Artifacts advertises filter) const back = { treeFilter: 'none' }; unwrap(await discover(unwrap(createTransport(filter.url, { tracer: new Tracer(), quirks: back })))); assert.equal(back.treeFilter, 'tree:0'); const art = { treeFilter: 'blob:none' }; unwrap(await discover(unwrap(createTransport(filter.url, { tracer: new Tracer(), quirks: art })))); assert.equal(art.treeFilter, 'blob:none'); }); test('limits hold on the unfiltered pack: limit_exceeded (pack bytes), never protocol; the filtered twin is under them', async () => { const { stock, filter } = twins(); const limits = { ...DEFAULT_LIMITS, maxPackBytes: 100_000 }; const o = () => ({ tracer: new Tracer(), limits }); const small = unwrap(await readFile(filter.url, { ref: 'main', path: 'README.md' }, o())); assert.equal((await drain(small.stream)).toString(), '# readme\n'); for (const [what, run] of [ ['readFile', () => readFile(stock.url, { ref: 'main', path: 'README.md' }, o())], ['readCommit', () => readCommit(stock.url, { ref: 'main' }, o())], ['readLog', () => readLog(stock.url, { ref: 'main', limit: 2 }, o())], ['checkoutTree', () => checkoutTree(stock.url, { ref: 'main' }, o())], ['editFile', () => editFile(stock.url, { path: 'a/b/c.txt', find: 'MARK', replace: 'X', author: A, message: 'm' }, o())], ]) { const r = await run(); assert.equal(r.ok, false, what); assert.equal(r.error.code, 'limit_exceeded', `${what}: ${r.error.code} ${r.error.message}`); assert.equal(r.error.what, 'pack bytes', what); } assert.equal(git(stock.bare, ['rev-parse', 'main']).trim(), seeded.main, 'nothing pushed'); }); test('a server that advertises filter and answers a filtered fetch with an empty body: protocol, the quirk unchanged', async () => { const { filter } = twins(); const proxy = await startProxy(srv.url, ctx => (ctx.stage === 'fetch' && /filter /.test(ctx.reqBody.toString('latin1')) ? { type: 'bytes', noForward: true, bytes: new Uint8Array(0) } : { type: 'pass' })); try { const t = traced(); const quirks = {}; const url = `${proxy.url}/${filter.url.split('/').pop()}`; const r = await readFile(url, { ref: 'main', path: 'a/b/c.txt' }, { tracer: t.tracer, quirks }); assert.equal(r.ok, false); assert.equal(r.error.code, 'protocol'); assert.equal(r.error.reason, 'empty_response'); assert.equal(r.error.message, 'v2: response ended without flush'); assert.equal(quirks.treeFilter, 'tree:0'); const probes = spansOf(t.ex).filter(s => s.name === 'git.upload-pack discover'); assert.equal(probes.length, 1); assert.equal(probes[0].attrs['git.server.filter'], true); // an empty answer to an unfiltered request is not a reason to probe: it is the protocol error, no advertisement read const t2 = traced(); const blob = git(filter.bare, ['rev-parse', 'main:a/b/c.txt']).trim(); const proxy2 = await startProxy(srv.url, ctx => (ctx.stage === 'fetch' && ctx.reqBody.length > 0 && !/command=ls-refs/.test(ctx.reqBody.toString('latin1')) ? { type: 'bytes', noForward: true, bytes: new Uint8Array(0) } : { type: 'pass' })); try { const tr = unwrap(createTransport(`${proxy2.url}/${filter.url.split('/').pop()}`, { tracer: t2.tracer })); const { fetchObject } = await import('../src/fetch.js'); // (the request that carries no argument a server may refuse: a want named a blob is sent no `deepen`) const { TYPE } = await import('../src/objects.js'); const got = await fetchObject(tr, { oid: blob, filter: '', expectType: TYPE.BLOB }); assert.equal(got.ok, false); assert.equal(got.error.code, 'protocol'); assert.equal(spansOf(t2.ex).filter(s => s.name === 'git.upload-pack discover').length, 0); // the same for a caller's own depth: it is sent as given and its refusal is the caller's answer const own = await fetchObject(tr, { oid: blob, filter: '', deepen: 0 }); assert.equal(own.ok, false); assert.equal(own.error.code, 'protocol'); assert.equal(spansOf(t2.ex).filter(s => s.name === 'git.upload-pack discover').length, 0); // A want of unknown type is sent `deepen 1` by default (fetchobject-deepen.test.js), an argument a server may not // take: its empty answer reads the advertisement, once per quirks record. It lists `shallow`, so the answer is // still the protocol error, and the next empty answer reads nothing. const typeless = await fetchObject(tr, { oid: blob, filter: '' }); assert.equal(typeless.ok, false); assert.equal(typeless.error.code, 'protocol'); assert.equal(typeless.error.reason, 'empty_response'); const probes2 = spansOf(t2.ex).filter(s => s.name === 'git.upload-pack discover'); assert.equal(probes2.length, 1); assert.equal(probes2[0].attrs['git.server.shallow'], true); const again = await fetchObject(tr, { oid: blob, filter: '' }); assert.equal(again.ok, false); assert.equal(again.error.code, 'protocol'); assert.equal(spansOf(t2.ex).filter(s => s.name === 'git.upload-pack discover').length, 1); } finally { await proxy2.close(); } } finally { await proxy.close(); } }); test('an advertisement the probe cannot read (cut inside a pkt) decides nothing: the empty response stays protocol', async () => { const { stock } = twins(); // GET info/refs (no request body) loses its last two bytes, inside the closing flush; the fetch is the stock server's own const proxy = await startProxy(srv.url, ctx => (ctx.reqBody.length === 0 && ctx.bytes.length > 0 ? { type: 'bytes', bytes: ctx.bytes.subarray(0, ctx.bytes.length - 2) } : { type: 'pass' })); try { const t = traced(); const quirks = {}; const r = await readFile(`${proxy.url}/${stock.url.split('/').pop()}`, { ref: 'main', path: 'a/b/c.txt' }, { tracer: t.tracer, quirks }); assert.equal(r.ok, false); assert.equal(r.error.code, 'protocol'); assert.equal(r.error.reason, 'empty_response'); assert.equal(quirks.treeFilter, 'tree:0', 'nothing learned'); const probe = spansOf(t.ex).filter(s => s.name === 'git.upload-pack discover'); assert.equal(probe.length, 1); assert.equal(probe[0].attrs['error.type'], 'protocol'); assert.equal(probe[0].status.message, 'trailing 2 bytes after last pkt-line'); } finally { await proxy.close(); } }); // #136: with no quirks passed, the no-filter learning is remembered per remote too: across any order of // edit/read/checkout/log/merge, at most one filtered request (the one the stock server dies on: empty_response) and one // advertisement probe per remote. Counted at the fetchImpl. A fresh `quirks: {}` per call (ply before #136) pays per call. const OPS = ['edit', 'read', 'checkout', 'log', 'merge']; const orders = [...OPS.map((_, i) => [...OPS.slice(i), ...OPS.slice(0, i)]), [...OPS].reverse()]; async function sweep(order, perCall) { const { stock } = twins(); let filtered = 0, probes = 0; const fetchImpl = (url, init) => { const u = String(url); if (u.endsWith('/git-upload-pack') && init?.body && Buffer.from(init.body).toString('latin1').includes('filter ')) filtered++; if (u.includes('/info/refs?service=git-upload-pack')) probes++; return globalThis.fetch(url, init); }; const ex = arrayExporter(); const o = () => ({ tracer: new Tracer({ exporter: ex }), fetchImpl, ...(perCall ? { quirks: {} } : {}) }); const run = { edit: async () => unwrap(await editFile(stock.url, { path: 'a/b/c.txt', find: 'MARK', replace: 'EDITED', author: A, message: 'edit', time: T }, { ...o(), verify: true })), read: async () => drain(unwrap(await readFile(stock.url, { path: 'big.bin' }, o())).stream), checkout: async () => plain(unwrap(await checkoutTree(stock.url, { ref: 'main' }, o()))), log: async () => unwrap(await readLog(stock.url, { ref: 'main', limit: 3 }, o())), merge: async () => unwrap(await mergeBranch(stock.url, { ref: 'refs/heads/main', from: 'refs/heads/feat', author: A, message: 'merge feat', time: T }, o())), }; for (const op of order) await run[op](); assert.deepEqual(fsckProblems(stock.bare), []); const empties = spansOf(ex).filter(s => s.name === 'git.upload-pack fetch' && s.attrs['error.kind'] === 'empty_response').length; return { filtered, probes, empties }; } test('#136: no quirks passed, any order: one empty_response and one advertisement probe per stock remote', async () => { forgetQuirks(); for (const order of orders) { const { filtered, probes, empties } = await sweep(order, false); assert.equal(filtered, 1, `${order}: one filtered request`); assert.equal(empties, 1, `${order}: one empty_response`); assert.equal(probes, 1, `${order}: one advertisement read`); } const before = await sweep(OPS, true); assert.ok(before.empties >= 4 && before.probes === before.empties, JSON.stringify(before)); });