GitHubivarvong

ply

main
ply6 git calls92 ms27 KiB3 cachedprotocol v2
216 lines12 KiBRaw
1// A stored transfer plan is untrusted input (#238 review): its temp ref is not a free field. The oracle is git
2// http-backend behind a proxy that counts requests, and git's own view of the destination's refs afterwards:
3// - a plan whose tempRef is pointed at an existing branch (refs/heads/victim), with a fake batch step at that
4// branch's commit and both hashes recomputed (they are unkeyed: anyone can), is refused as bad_request by
5// checkTransferPlan, transferStatus, transferStep and runTransfer before any request is sent, and the branch is
6// still where git put it
7// - a temp ref under the transfer prefix that is not the one the plan's key names is refused the same way
8// - a plan for another ref than the one the caller names on the call is refused the same way, and so is every call
9// that names no ref: the caller, never the stored plan, says which ref the pushes create
10// - each refusal is by the rule it states (the error's `what`: tempRef, ref, plan, commit)
11import { test, beforeAll as before, afterAll as after } from 'vitest';
12import assert from 'node:assert/strict';
13import http from 'node:http';
14import { execFileSync } from 'node:child_process';
15import { mkdirSync, writeFileSync } from 'node:fs';
16import { join } from 'node:path';
17import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, initRepo } from './server.js';
18import { scanPackIndex } from '../src/packscanindex.js';
19import { resolvePackRange } from '../src/packrangeresolve.js';
20import { rangeFromBytes } from '../src/rangesource.js';
21import { createTransport } from '../src/http.js';
22import { planHash } from '../src/packplan.js';
23import { planTransfer, checkTransferPlan, transferPlanHash } from '../src/transferplan.js';
24import { transferStatus, transferStep, runTransfer } from '../src/transfer.js';
25import { unwrap } from './unwrap.js';
27const ident = ['-c', 'user.name=Ann Author', '-c', 'user.email=ann@x'];
28const BUDGET = { maxRequestBytes: 200_000, maxObjects: 300, maxObjectBytes: 1 << 20, maxCompressedObjectBytes: 1 << 20, reserveBytes: 0, reserveObjects: 0 };
29const COPY = 'refs/heads/copy';
30const VICTIM = 'refs/heads/victim';
31const streamOf = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
33// A proxy in front of the git server that counts the requests it is sent.
34function countingProxy(upstream) {
35 let count = 0;
36 const server = http.createServer((req, res) => {
37 count++;
38 const to = new URL(req.url, upstream);
39 const up = http.request(to, { method: req.method, headers: { ...req.headers, host: to.host } }, r => { res.writeHead(r.statusCode, r.headers); r.pipe(res); });
40 req.pipe(up);
41 });
42 return new Promise(resolve => { server.listen(0, '127.0.0.1', () => resolve({ url: `http://127.0.0.1:${server.address().port}`, count: () => count, close: () => new Promise(done => { server.close(done); }) })); });
45let root, cleanup, srv, proxy, from, commit, n = 0;
46before(async () => {
47 ({ root, cleanup } = tmpRoot());
48 srv = await startGitServer(root);
49 proxy = await countingProxy(srv.url);
50 const work = join(root, 'source');
51 mkdirSync(join(work, 'a'), { recursive: true });
52 initRepo(work);
53 writeFileSync(join(work, 'a', 'one.txt'), 'one\n');
54 writeFileSync(join(work, 'two.txt'), 'two\n');
55 git(work, ['add', '.']);
56 git(work, [...ident, 'commit', '-q', '-m', 'source']);
57 const head = git(work, ['rev-parse', 'HEAD']).trim();
58 const pack = new Uint8Array(gitBuf(work, ['pack-objects', '--stdout', '-q'], git(work, ['rev-list', '--objects', '--no-walk', head])));
59 commit = new Uint8Array(gitBuf(work, ['cat-file', 'commit', head]));
60 const scanned = unwrap(await scanPackIndex(streamOf(pack)));
61 const source = rangeFromBytes(pack);
62 from = { source, pack: scanned, resolved: unwrap(await resolvePackRange(scanned, source)) };
63});
64after(async () => { await proxy?.close(); await srv?.close(); cleanup?.(); });
66// A destination with a branch the transfer has no business with: an unrelated history, pushed by git.
67function destination() {
68 const name = `d${n++}`;
69 const bare = makeBareRepo(root, name);
70 const other = join(root, `${name}-other`);
71 mkdirSync(other);
72 initRepo(other);
73 writeFileSync(join(other, 'precious.txt'), 'only copy\n');
74 git(other, ['add', '.']);
75 git(other, [...ident, 'commit', '-q', '-m', 'victim']);
76 git(other, ['push', '-q', bare, `HEAD:${VICTIM}`]);
77 return { bare, victim: git(bare, ['rev-parse', VICTIM]).trim(), t: unwrap(createTransport(`${proxy.url}/${name}.git`)) };
79const refsOf = bare => git(bare, ['for-each-ref', '--format=%(refname) %(objectname)']).trim();
80// Both hashes recomputed over the edited plan, as a writer of the plan store can.
81function rehashed(plan) {
82 const { hash: _packHash, ...packBody } = plan.pack;
83 plan.pack.hash = planHash(packBody);
84 const { hash: _hash, ...body } = plan;
85 plan.hash = transferPlanHash(body);
86 return plan;
88// Every entry point refuses `plan` with bad_request and the stated `what` (which rule refused it), sends nothing, and
89// leaves the destination's refs as git shows them. Each call names the ref, as every stored-plan call must.
90const answerOf = r => (r.ok ? JSON.stringify(r.value) : [r.error.code, r.error.what]);
91async function refusedUnsent(plan, what, opts = { ref: COPY }) {
92 const { bare, victim, t } = destination();
93 const before = proxy.count();
94 const answers = [
95 checkTransferPlan(plan, opts),
96 await transferStatus(t, plan, opts),
97 await transferStep(t, from, plan, 0, { ...opts, commit }),
98 await transferStep(t, from, plan, 1, { ...opts, commit }),
99 await runTransfer(t, from, plan, commit, opts),
100 ];
101 for (const [i, r] of answers.entries()) assert.deepEqual(answerOf(r), ['bad_request', what], `entry point ${i}`);
102 assert.equal(proxy.count() - before, 0, 'no request was sent');
103 assert.equal(refsOf(bare), `${VICTIM} ${victim}`, 'the destination is as git left it');
105const planned = (opts = {}) => unwrap(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: COPY, commit, ...opts }));
107test('a stored plan whose temp ref is an existing branch, hashes recomputed, is refused before any request and the branch stays', async () => {
108 const honest = planned();
109 const { victim } = destination();
110 const stored = structuredClone(honest);
111 stored.tempRef = VICTIM;
112 stored.steps = [{ kind: 'batch', temp: victim, prev: { tag: 'none' } }, { kind: 'final', prev: { tag: 'some', value: victim } }];
113 stored.pack.batches = [{ kind: 'blobs', entries: [], packBytes: 32, objects: 0, refRewrites: 0 }, ...stored.pack.batches];
114 rehashed(stored);
115 await refusedUnsent(stored, 'tempRef');
116});
118test('a temp ref under the prefix that the plan\'s key does not name, and a plan for another ref than the caller names, are refused unsent', async () => {
119 const honest = planned();
120 const moved = structuredClone(honest);
121 moved.tempRef = `refs/ply/transfer/${'a'.repeat(40)}`;
122 await refusedUnsent(rehashed(moved), 'tempRef');
123 // the honest plan is for refs/heads/copy; this call names another ref
124 assert.ok(checkTransferPlan(structuredClone(honest), { ref: COPY }).ok);
125 await refusedUnsent(structuredClone(honest), 'ref', { ref: 'refs/heads/elsewhere' });
126 // the store's plan swapped for a whole, self-consistent one that creates another ref (planTransfer made it: every
127 // hash, key and temp ref is right); the caller still names refs/heads/copy
128 for (const ref of ['refs/heads/main', 'refs/tags/v9.9.9']) await refusedUnsent(planned({ ref }), 'ref');
129 // a plan made with another temp prefix is this plan only to a caller that names that prefix
130 const custom = planned({ tempPrefix: 'refs/x/y/' });
131 await refusedUnsent(custom, 'tempRef');
132 assert.ok(checkTransferPlan(custom, { ref: COPY, tempPrefix: 'refs/x/y/' }).ok);
133 const { bare, t } = destination();
134 assert.equal(unwrap(await runTransfer(t, from, custom, commit, { ref: COPY, tempPrefix: 'refs/x/y/' })).tag, 'done');
135 assert.equal(git(bare, ['rev-parse', COPY]).trim(), custom.head);
136});
138test('a call that does not name the ref is refused unsent: a stored plan never chooses the ref its pushes create', async () => {
139 // owner decision: transferStatus, transferStep and runTransfer take the destination ref from the caller on every
140 // call. The plan here is honest and would run, so the only reason to refuse is the missing name
141 const honest = planned();
142 const { bare, victim, t } = destination();
143 const before = proxy.count();
144 const answers = [
145 checkTransferPlan(honest), checkTransferPlan(honest, {}), checkTransferPlan(honest, { ref: '' }), checkTransferPlan(honest, { ref: 5 }), checkTransferPlan(honest, null),
146 await transferStatus(t, honest), await transferStatus(t, honest, {}),
147 // the bare-bytes form of the last argument names no ref
148 await transferStep(t, from, honest, 0, commit), await transferStep(t, from, honest, 0, { commit }),
149 await runTransfer(t, from, honest, commit), await runTransfer(t, from, honest, commit, { maxSteps: 1 }),
150 ];
151 for (const [i, r] of answers.entries()) assert.deepEqual(answerOf(r), ['bad_request', 'ref'], `call ${i}`);
152 assert.equal(proxy.count() - before, 0, 'no request was sent');
153 assert.equal(refsOf(bare), `${VICTIM} ${victim}`, 'the destination is as git left it');
154});
156test('a `commit` that is not a Uint8Array is refused unsent by runTransfer and transferStep, naming `commit`: neither throws', async () => {
157 // a resuming caller reads the source commit's bytes back from its store next to the plan. The plan here is honest
158 // and the ref is named, so the only reason to refuse is the argument. Expected value, by the contract and not from
159 // the code: a Result API answers a wrong argument with a coded error (bad_request, what 'commit'), before any use
160 const honest = planned();
161 const { bare, victim, t } = destination();
162 const before = proxy.count();
163 const notBytes = {
164 null: null,
165 'not given': void 0,
166 'a string': 'junk',
167 'a number': 7,
168 'an object': {},
169 'an array of the bytes': [...commit],
170 'an ArrayBuffer of the bytes': commit.buffer.slice(commit.byteOffset, commit.byteOffset + commit.byteLength),
171 };
172 const wrong = [];
173 for (const [label, value] of Object.entries(notBytes)) {
174 const calls = {
175 runTransfer: () => runTransfer(t, from, honest, value, { ref: COPY }),
176 transferStep: () => transferStep(t, from, honest, 0, { commit: value, ref: COPY }),
177 };
178 for (const [name, call] of Object.entries(calls)) {
179 let got;
180 try {
181 got = JSON.stringify(answerOf(await call()));
182 } catch (e) {
183 got = `threw ${String(e)}`;
184 }
185 if (got !== '["bad_request","commit"]') wrong.push(`${name}(commit = ${label}): ${got}`);
186 }
187 }
188 assert.deepEqual(wrong, []);
189 assert.equal(proxy.count() - before, 0, 'no request was sent');
190 assert.equal(refsOf(bare), `${VICTIM} ${victim}`, 'the destination is as git left it');
191});
193test('on a finished destination a plan that does not pass is still refused, not answered done; a commit that does not give the planned head is refused unsent', async () => {
194 const honest = planned();
195 const { bare, victim, t } = destination();
196 assert.equal(unwrap(await runTransfer(t, from, honest, commit, { ref: COPY })).tag, 'done');
197 const finished = refsOf(bare);
198 assert.equal(finished, `${COPY} ${honest.head}\n${VICTIM} ${victim}`);
199 const before = proxy.count();
200 const badHash = { ...structuredClone(honest), hash: 'f'.repeat(40) };
201 const otherRef = { ...structuredClone(honest), ref: 'refs/heads/other' };
202 const { pack: _pack, ...noPack } = structuredClone(honest);
203 const { steps: _steps, ...noSteps } = structuredClone(honest);
204 for (const [label, plan] of [['hash', badHash], ['ref edited', otherRef], ['no pack', noPack], ['no steps', noSteps]]) {
205 for (const r of [await transferStatus(t, plan, { ref: COPY }), await runTransfer(t, from, plan, commit, { ref: COPY })]) {
206 assert.deepEqual(answerOf(r), ['bad_request', 'plan'], label);
207 }
208 }
209 // the source commit with more message is another commit: git hash-object gives its id, which is not the planned head
210 const other = new Uint8Array([...commit, ...new TextEncoder().encode('more\n')]);
211 const otherHead = execFileSync('git', ['hash-object', '-t', 'commit', '--stdin'], { input: other, encoding: 'utf8' }).trim();
212 assert.notEqual(otherHead, honest.head);
213 assert.deepEqual(answerOf(await runTransfer(t, from, honest, other, { ref: COPY })), ['bad_request', 'commit']);
214 assert.equal(proxy.count() - before, 0, 'no request was sent');
215 assert.equal(refsOf(bare), finished);
216});
Diagnostics6 git calls · 92 ms
CallStatusSizeTime092 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=11990
git.response.cache=UPDATING
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=81
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2004.1 KiB81 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=4213
git.response.cache=MISS
git.response.duration_ms=81
git.fetch.wants=a4b5eca81509c43f6a06edddae257da8cca3893a
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4213
git.pkt.payload.max=4186
git.sideband.band1.bytes=4186
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2bb3d59b9cfc3
Request
141f5d4e…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format