GitHubivarvong
ply
main
1// A stored transfer plan is untrusted input (#238 review): the check is total and covers every field a step reads.
2// The oracle is git http-backend behind a proxy that counts requests, and git's own view of the destination's refs:
3// - a generated corpus (every field of an honest plan, at every depth, deleted or replaced by a value of another
4// type) is refused as bad_request by checkTransferPlan, transferStatus, transferStep and runTransfer; none throws
5// and none sends a request. Expected value, by hand: each of these plans differs from the honest one in a field
6// the plan's hashes cover, so none of them is the plan that was made
7// - the same for right-typed edits with both hashes left as they were (a batch entry dropped, crossBatchRef flipped,
8// the budget, the trailer, the selection, a batch's byte count)
9// - the honest plan still runs: git shows the new ref at the source commit's tree
10import { test, beforeAll as before, afterAll as after } from 'vitest';
11import assert from 'node:assert/strict';
12import http from 'node:http';
13import { mkdirSync, writeFileSync } from 'node:fs';
14import { join } from 'node:path';
15import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, initRepo } from './server.js';
16import { scanPackIndex } from '../src/packscanindex.js';
17import { resolvePackRange } from '../src/packrangeresolve.js';
18import { rangeFromBytes } from '../src/rangesource.js';
19import { createTransport } from '../src/http.js';
20import { planTransfer, checkTransferPlan } from '../src/transferplan.js';
21import { transferStatus, transferStep, runTransfer } from '../src/transfer.js';
22import { unwrap } from './unwrap.js';
24const ident = ['-c', 'user.name=Ann Author', '-c', 'user.email=ann@x'];
25const BUDGET = { maxRequestBytes: 200_000, maxObjects: 300, maxObjectBytes: 1 << 20, maxCompressedObjectBytes: 1 << 20, reserveBytes: 0, reserveObjects: 0 };
26const COPY = 'refs/heads/copy';
27const streamOf = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
29// A proxy in front of the git server that counts the requests it is sent.
30function countingProxy(upstream) {
31 let count = 0;
32 const server = http.createServer((req, res) => {
33 count++;
34 const to = new URL(req.url, upstream);
35 const up = http.request(to, { method: req.method, headers: { ...req.headers, host: to.host } }, r => { res.writeHead(r.statusCode, r.headers); r.pipe(res); });
36 req.pipe(up);
37 });
38 return new Promise(resolve => { server.listen(0, '127.0.0.1', () => resolve({ url: `http://127.0.0.1:${server.address().port}`, count: () => count, close: () => new Promise(done => { server.close(done); }) })); });
39}
41let root, cleanup, srv, proxy, from, commit, sourceTree, honest, n = 0;
42before(async () => {
43 ({ root, cleanup } = tmpRoot());
44 srv = await startGitServer(root);
45 proxy = await countingProxy(srv.url);
46 const work = join(root, 'source');
47 mkdirSync(join(work, 'a'), { recursive: true });
48 initRepo(work);
49 writeFileSync(join(work, 'a', 'one.txt'), 'one\n');
50 writeFileSync(join(work, 'two.txt'), 'two\n');
51 git(work, ['add', '.']);
52 git(work, [...ident, 'commit', '-q', '-m', 'source']);
53 const head = git(work, ['rev-parse', 'HEAD']).trim();
54 sourceTree = git(work, ['rev-parse', 'HEAD^{tree}']).trim();
55 const pack = new Uint8Array(gitBuf(work, ['pack-objects', '--stdout', '-q'], git(work, ['rev-list', '--objects', '--no-walk', head])));
56 commit = new Uint8Array(gitBuf(work, ['cat-file', 'commit', head]));
57 const scanned = unwrap(await scanPackIndex(streamOf(pack)));
58 const source = rangeFromBytes(pack);
59 from = { source, pack: scanned, resolved: unwrap(await resolvePackRange(scanned, source)) };
60 honest = unwrap(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: COPY, commit }));
61});
62after(async () => { await proxy?.close(); await srv?.close(); cleanup?.(); });
64function destination() {
65 const name = `d${n++}`;
66 const bare = makeBareRepo(root, name);
67 return { bare, t: unwrap(createTransport(`${proxy.url}/${name}.git`)) };
68}
69const refsOf = bare => git(bare, ['for-each-ref', '--format=%(refname) %(objectname)']).trim();
71// What each entry point answers for `plan`: the error code, or what it did instead (an ok value, a throw).
72async function answers(t, plan) {
73 const calls = [
74 ['checkTransferPlan', () => checkTransferPlan(plan, { ref: COPY })],
75 ['transferStatus', () => transferStatus(t, plan, { ref: COPY })],
76 ['transferStep', () => transferStep(t, from, plan, 0, { commit: commit, ref: COPY })],
77 ['runTransfer', () => runTransfer(t, from, plan, commit, { maxSteps: 1, retries: 0, ref: COPY })],
78 ];
79 const out = [];
80 for (const [name, call] of calls) {
81 try {
82 const r = await call();
83 out.push([name, r.ok ? `ok ${JSON.stringify(r.value)}` : r.error.code]);
84 } catch (e) {
85 out.push([name, `threw ${String(e)}`]);
86 }
87 }
88 return out;
89}
90// Every plan in `plans` ([label, plan]) is refused by every entry point; nothing is sent; the destination has no refs.
91async function allRefusedUnsent(plans) {
92 const { bare, t } = destination();
93 const before = proxy.count();
94 const wrong = [];
95 for (const [label, plan] of plans) {
96 for (const [name, got] of await answers(t, plan)) if (got !== 'bad_request') wrong.push(`${label}: ${name}: ${got}`);
97 }
98 assert.deepEqual({ count: wrong.length, first: wrong.slice(0, 8) }, { count: 0, first: [] });
99 assert.equal(proxy.count() - before, 0, 'no request was sent');
100 assert.equal(refsOf(bare), '', 'the destination has no refs');
101}
103// Every path into a JSON value: its keys and indices from the top.
104function* pathsOf(v, here = []) {
105 if (typeof v !== 'object' || v === null) return;
106 for (const k of Object.keys(v)) {
107 yield [...here, k];
108 yield* pathsOf(v[k], [...here, k]);
109 }
110}
111const DELETE = Symbol('delete');
112// Values of every JSON type, with the near misses of the plan's own shapes (a negative or fractional count, an Option
113// without a value, a list holding null).
114const WRONG = [DELETE, null, 0, -1, 1.5, 'x', true, [], {}, [null], { tag: 'some' }, { tag: 'some', value: 7 }];
115function mutated(plan, path, value) {
116 const copy = structuredClone(plan);
117 let o = copy;
118 for (const k of path.slice(0, -1)) o = o[k];
119 const last = path.at(-1);
120 if (value !== DELETE) o[last] = value;
121 else if (Array.isArray(o)) o.splice(Number(last), 1);
122 else delete o[last];
123 return copy;
124}
126test('a generated corpus of missing and wrong-type fields is refused with bad_request by every entry point: no throw, no request', async () => {
127 const same = JSON.stringify(honest);
128 const corpus = [];
129 for (const path of pathsOf(honest)) {
130 for (const value of WRONG) {
131 const plan = mutated(honest, path, value);
132 // a value that is the honest one (0 for a zero reserve) is not a mutation
133 if (JSON.stringify(plan) !== same) corpus.push([`${path.join('.')} = ${value === DELETE ? 'deleted' : JSON.stringify(value)}`, plan]);
134 }
135 }
136 for (const value of [null, 0, 'x', true, [], {}]) corpus.push([`plan = ${JSON.stringify(value)}`, value]);
137 assert.ok(corpus.length > 300, `the corpus covers the plan (${corpus.length} plans)`);
138 await allRefusedUnsent(corpus);
139});
141test('a right-typed edit of any field a step reads, both hashes left as stored, is refused unsent', async () => {
142 const edit = (label, change) => {
143 const plan = structuredClone(honest);
144 change(plan);
145 return [label, plan];
146 };
147 const final = p => p.pack.batches.at(-1);
148 await allRefusedUnsent([
149 edit('a final-batch entry dropped', p => { final(p).entries.pop(); }),
150 edit('the final batch emptied', p => { final(p).entries = []; }),
151 edit('crossBatchRef flipped', p => { p.pack.crossBatchRef = !p.pack.crossBatchRef; }),
152 edit('budget raised', p => { p.pack.budget.maxRequestBytes += 1; }),
153 edit('trailer replaced', p => { p.pack.trailer = 'f'.repeat(40); }),
154 edit('selection shortened', p => { p.pack.select.pop(); }),
155 edit('a batch byte count changed', p => { final(p).packBytes += 1; }),
156 edit('atomic turned off', p => { p.atomic = false; }),
157 ]);
158});
160test('the honest plan, read back from JSON, passes the check and runs: git shows the ref at the source tree', async () => {
161 const stored = JSON.parse(JSON.stringify(honest));
162 assert.ok(checkTransferPlan(stored, { ref: COPY }).ok);
163 const { bare, t } = destination();
164 const run = unwrap(await runTransfer(t, from, stored, commit, { ref: COPY }));
165 assert.equal(run.tag, 'done');
166 assert.equal(git(bare, ['rev-parse', `${COPY}^{tree}`]).trim(), sourceTree);
167 git(bare, ['fsck', '--strict']);
168});
Diagnostics6 git calls · 115 ms
CallStatusSizeTime0115 ms
discover200191 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB · hit23 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=21 git.response.bytes=11990 git.response.cache=HIT git.response.duration_ms=23 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetch2003.3 KiB85 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=85 git.response.bytes=3392 git.response.cache=MISS git.response.duration_ms=85 git.fetch.wants=e493deae91dad221564b38db8ade0e67096b263b git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3392 git.pkt.payload.max=3365 git.sideband.band1.bytes=3365 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e1a53e09fc5e44
- Request
- be75f764…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format