GitHubivarvong
ply
main
1// README "Custom exchanges on a transport" (#239): the `// custom-exchange:` block is read from the README and executed
2// as written (imports bound to src/: 'ply' the package entry, 'ply/transport', 'ply/push') against `git http-backend`
3// behind an auth gate, and stock git judges what it left (rev-parse, cat-file, fsck --strict). Then the error paths an
4// exchange on `frames` meets, each a typed PlyError from the iteration, with the span and the credential checked:
5// wrong credential (401; the server echoes the credential back) · no credential · a credential that would send nothing ·
6// a body stream that errors midway · a stream that yields more or fewer bytes than `length` (measured: frames does not
7// check it, the README says so) · a receive-pack refusal (ng) · a corrupt pack (unpack failed) · a server that is not
8// git (foreign content type) · an aborted signal
9// test/types/custom-exchange.typetest.ts is the block as TypeScript (UPDATE_README_TYPETEST=1 rewrites it).
10import { test, beforeAll as before, afterAll as after } from 'vitest';
11import assert from 'node:assert/strict';
12import { readFileSync, writeFileSync } from 'node:fs';
13import * as ply from '../src/index.js';
14import * as plyTransport from '../src/transport.js';
15import * as plyPush from '../src/push.js';
16import { isPlyError } from '../src/errors.js';
17import { git, tmpRoot, makeBareRepo, startGitServer, fsckProblems } from './server.js';
19const README = readFileSync(new URL('../README.md', import.meta.url), 'utf8');
20const TYPETEST_PATH = new URL('./types/custom-exchange.typetest.ts', import.meta.url);
21const MODULES = { ply, 'ply/transport': plyTransport, 'ply/push': plyPush };
22const TS_SPECIFIER = { ply: '../../src/index.js', 'ply/transport': '../../src/transport.js', 'ply/push': '../../src/push.js' };
23const BLOCK = /```js\n\/\/ custom-exchange: ([\w-]+)\n([\s\S]*?)```/g;
24const IMPORT = /^import \{([^}]+)\} from '([^']+)';\n/gm;
25const blocks = [...README.matchAll(BLOCK)].map(([, step, code]) => ({ step, code }));
26const AsyncFunction = (async () => {}).constructor;
27// The block as a function of its free variables (url, opts); returns every top-level declaration.
28function compile(code) {
29 const imports = [...code.matchAll(IMPORT)].map(([, names, from]) => ({ names: names.split(',').map(s => s.trim()).filter(Boolean), from }));
30 const body = code.replace(IMPORT, '');
31 const declared = [...body.matchAll(/^(?:const|let) (?:\{ (\w+) \}|(\w+))/gm)].map(m => m[1] ?? m[2]);
32 const bound = {};
33 for (const { names, from } of imports) for (const n of names) bound[n] = MODULES[from][n];
34 return async scope => {
35 const all = { ...bound, ...scope };
36 const fn = new AsyncFunction(...Object.keys(all), `${body}\nreturn { ${declared.join(', ')} };`);
37 return fn(...Object.values(all));
38 };
39}
41const TOKEN = 'ghp_customExchangeToken0123456789abcdef';
42const WRONG = 'ghp_wrongToken9876543210fedcbaFEDCBA00';
43const basic = tok => `Basic ${Buffer.from(`x-access-token:${tok}`).toString('base64')}`;
44let tmp, srv;
45const seen = [];
46// The auth gate in front of git http-backend: the right Basic credential passes; anything else is a 401 whose body
47// echoes the Authorization it got (a careless server), so a leak of the credential into an error would show.
48const gate = async (url, init) => {
49 const h = new Headers(init?.headers);
50 const got = h.get('authorization');
51 seen.push({ url: String(url), auth: got, aborted: init?.signal?.aborted === true });
52 if (got !== basic(TOKEN)) return new Response(`denied: ${got}\n`, { status: 401, headers: { 'www-authenticate': 'Basic realm="git"', 'content-type': 'text/plain' } });
53 return fetch(url, init);
54};
55before(async () => { tmp = tmpRoot(); srv = await startGitServer(tmp.root); });
56after(async () => { await srv?.close(); tmp?.cleanup(); });
57let n = 0;
58const repo = () => { const name = `cx${n++}`; return { bare: makeBareRepo(tmp.root, name), url: `${srv.url}/${name}.git` }; };
59const leaks = (value, ...secrets) => { const text = JSON.stringify(value) + String(value?.message ?? '') + String(value?.stack ?? ''); return secrets.filter(s => text.includes(s) || text.includes(Buffer.from(`x-access-token:${s}`).toString('base64'))); };
60const rev = (bare, r) => { try { return git(bare, ['rev-parse', '--verify', '-q', r]).trim(); } catch { return null; } };
62test('the README has exactly one custom-exchange block, and its TypeScript twin is current', () => {
63 assert.deepEqual(blocks.map(b => b.step), ['receive-pack']);
64 const head = '// GENERATED from README.md "Custom exchanges on a transport" by test/custom-exchange.test.js (UPDATE_README_TYPETEST=1\n'
65 + '// rewrites it). The block as a TypeScript consumer writes it: `npm run typecheck` compiles it.\n'
66 + "const url: string = 'https://git.example.com/acme/site.git';\nconst opts = { auth: { token: process.env['GIT_TOKEN'] ?? '' } };\n";
67 // a TypeScript consumer annotates two lets: the span list, and the state that widens (streamInit gives the Reading it
68 // starts in)
69 const ts = head + blocks[0].code
70 .replace(IMPORT, (_, names, from) => `import {${names}} from '${TS_SPECIFIER[from] ?? from}';\n`)
71 .replace("from '../../src/push.js';\n", "from '../../src/push.js';\nimport type { StreamState } from '../../src/push.js';\n")
72 .replace("from '../../src/index.js';\n", "from '../../src/index.js';\nimport type { EndedSpan } from '../../src/index.js';\n")
73 .replace('const spans = [];', 'const spans: EndedSpan[] = [];')
74 .replace('let reply = streamInit(', 'let reply: StreamState = streamInit(');
75 if (process.env.UPDATE_README_TYPETEST) writeFileSync(TYPETEST_PATH, ts);
76 assert.equal(readFileSync(TYPETEST_PATH, 'utf8'), ts, 'test/types/custom-exchange.typetest.ts drifted from the README (UPDATE_README_TYPETEST=1)');
77});
79test('happy path: the block, as written, lands a root commit through the auth gate; git agrees; the span carries the bytes', async () => {
80 const run = compile(blocks[0].code);
81 const { bare, url } = repo();
82 seen.length = 0;
83 const s = await run({ url, opts: { auth: { token: TOKEN }, fetchImpl: gate } });
84 console.log(JSON.stringify({ report: s.report, length: s.body.length, spans: s.spans.map(x => ({ name: x.name, status: x.status, attrs: Object.fromEntries(Object.entries(x.attrs).filter(([k]) => /^(git\.(request|response|service)|http\.(request\.method|response\.status_code))/.test(k))) })) }));
85 assert.deepEqual(s.report, { tag: 'statuses', refs: [{ ref: 'refs/heads/main', ok: true, reason: '' }] });
86 // stock git is the judge
87 assert.equal(rev(bare, 'refs/heads/main'), s.commit);
88 assert.equal(git(bare, ['cat-file', '-p', 'main:hello.txt']), 'hello\n');
89 assert.equal(git(bare, ['log', '-1', '--format=%an <%ae> %at%n%s', 'main']).trim(), 'Me <me@example.com> 1700000000\nhello');
90 assert.deepEqual(fsckProblems(bare), []);
91 // the span: what frames() filled in
92 const span = s.spans.find(x => x.name === 'custom receive-pack');
93 assert.ok(span, 'the caller span ended');
94 assert.equal(span.attrs['git.request.bytes'], s.body.length, 'git.request.bytes is the StreamBody length');
95 assert.equal(span.attrs['http.request.body.size'], s.body.length);
96 assert.equal(span.attrs['git.service'], 'git-receive-pack');
97 assert.equal(span.attrs['http.request.method'], 'POST');
98 assert.equal(span.attrs['http.response.status_code'], 200);
99 assert.ok(span.attrs['git.response.bytes'] > 0);
100 assert.ok(span.attrs['git.response.first_byte_ms'] >= 0);
101 assert.notEqual(span.status?.code, 2, 'not an error span');
102 // the credential went on the wire, and nowhere else
103 assert.ok(seen.every(x => x.auth === basic(TOKEN)), 'every request carried the credential');
104 assert.deepEqual(leaks(s.spans, TOKEN), [], 'no form of the credential in a span');
105});
107test('wrong credential: auth (401), typed, thrown from the iteration; the echoed credential is scrubbed; nothing lands', async () => {
108 const run = compile(blocks[0].code);
109 const { bare, url } = repo();
110 const spans = [];
111 const e = await run({ url, opts: { auth: { token: WRONG }, fetchImpl: gate } }).then(() => null, x => x);
112 assert.ok(isPlyError(e), String(e));
113 console.log(JSON.stringify({ code: e.code, status: e.status, message: e.message }));
114 assert.equal(e.code, 'auth');
115 assert.deepEqual(leaks(e, WRONG), [], 'the credential the server echoed is scrubbed from the error');
116 assert.equal(rev(bare, 'refs/heads/main'), null, 'nothing landed');
117 // the same through a span we keep: the error is recorded, scrubbed
118 const t = ply.createTransport(url, { auth: { token: WRONG }, fetchImpl: gate, tracer: new ply.Tracer({ onEnd: x => spans.push(x) }) });
119 const span = t.value.tracer.start('custom', { kind: ply.KIND.CLIENT, scrub: t.value.scrub });
120 const err2 = await (async () => { try { for await (const _ of plyTransport.frames(t.value, 'git-receive-pack', new Uint8Array(4).fill(48), span)) { /* none */ } } catch (x) { span.error(x); return x; } finally { span.end(); } return null; })();
121 assert.equal(err2.code, 'auth');
122 assert.equal(spans[0].status.code, 2, 'error status on the span');
123 assert.equal(spans[0].attrs['http.response.status_code'], 401);
124 assert.deepEqual(leaks(spans, WRONG), [], 'no form of the credential in the span');
125});
127test('no credential at all: the gate\'s 401 is auth; a credential that would send nothing is bad_request before any request', async () => {
128 const { url } = repo();
129 const t = ply.createTransport(url, { fetchImpl: gate });
130 const span = t.value.tracer.start('custom');
131 const e = await (async () => { try { for await (const _ of plyTransport.frames(t.value, 'git-receive-pack', new Uint8Array(4).fill(48), span)) { /* none */ } } catch (x) { return x; } return null; })();
132 assert.equal(e.code, 'auth');
133 const before = seen.length;
134 const bad = ply.createTransport(url, { auth: {}, fetchImpl: gate });
135 assert.equal(bad.ok, false);
136 assert.equal(bad.error.code, 'bad_request');
137 assert.equal(seen.length, before, 'no request was made');
138});
140// A direct exchange on the gate with a body of our choosing: the error, the span, and the ref afterwards.
141async function exchange(url, body, { signal = null } = {}) {
142 const spans = [];
143 const t = ply.createTransport(url, { auth: { token: TOKEN }, fetchImpl: gate, tracer: new ply.Tracer({ onEnd: x => spans.push(x) }), ...(signal ? { signal } : {}) });
144 const span = t.value.tracer.start('custom receive-pack', { kind: ply.KIND.CLIENT, scrub: t.value.scrub });
145 let reply = plyPush.streamInit(true);
146 let error = null;
147 try {
148 for await (const f of plyTransport.frames(t.value, 'git-receive-pack', body, span)) {
149 const st = plyPush.streamStep(reply, f);
150 if (!st.ok) throw st.error;
151 reply = st.value;
152 }
153 } catch (e) { error = e; span.error(e); } finally { span.end(); }
154 return { error, spans, done: error === null ? plyPush.streamFinish(reply) : null };
155}
156async function rootCommit(text) {
157 const file = new TextEncoder().encode(text);
158 const tree = ply.encodeTree([{ mode: '100644', name: 'f', oid: ply.hashObject('blob', file) }]).value;
159 const c = ply.encodeCommit({ tree: ply.hashObject('tree', tree), author: 'A <a@x> 1700000000 +0000', message: `${text}\n` }).value;
160 const commit = ply.hashObject('commit', c);
161 const { pack } = await ply.buildPack([{ type: ply.TYPE.BLOB, data: file }, { type: ply.TYPE.TREE, data: tree }, { type: ply.TYPE.COMMIT, data: c }]);
162 return { commit, pack };
163}
164const prefixOf = commands => plyPush.pushBody({ commands, caps: ['report-status', 'side-band-64k'] }).value;
166test('a body stream that errors midway: a typed error from the iteration, the span records it, nothing lands', async () => {
167 const { bare, url } = repo();
168 const { commit, pack } = await rootCommit('x');
169 const prefix = prefixOf([{ ref: 'refs/heads/main', oldOid: ply.ZERO_OID, newOid: commit }]);
170 const boom = new Error('pack source died');
171 let sent = 0;
172 const stream = new ReadableStream({ pull(c) { if (sent++ === 0) c.enqueue(prefix); else c.error(boom); } });
173 const r = await exchange(url, { stream, length: prefix.length + pack.length });
174 console.log(JSON.stringify({ code: r.error?.code, message: r.error?.message }));
175 assert.ok(isPlyError(r.error), String(r.error));
176 // undici aborts the request and fetch rejects 'fetch failed': frames reports a network error; the source's own error
177 // is on the cause chain. The README tells the caller to keep its own failure (as pushPack's streamSource does).
178 assert.equal(r.error.code, 'network');
179 const chain = []; for (let c = r.error; c; c = c.cause) chain.push(c);
180 assert.ok(chain.includes(boom), 'the source error is on the cause chain');
181 assert.equal(r.spans[0].status.code, 2);
182 assert.equal(r.spans[0].attrs['git.request.bytes'], prefix.length + pack.length);
183 assert.equal(rev(bare, 'refs/heads/main'), null);
184});
186// `length` is what the span reports (git.request.bytes); frames sends what the stream yields, chunked, and does not count
187// it. The README says so, and says what checks it: pushPack's streamSource (packwrite.ts). Here is what happens when a
188// raw exchange gets it wrong, so the documented behavior is a measured one.
189test('length is the span\'s figure, not a check: a longer stream is sent as yielded; a shorter one is a pack git refuses', async () => {
190 const observed = {};
191 for (const [name, extra] of [['longer', 7], ['shorter', -7]]) {
192 const { bare, url } = repo();
193 const { commit, pack } = await rootCommit(`len-${name}`);
194 const commands = [{ ref: 'refs/heads/main', oldOid: ply.ZERO_OID, newOid: commit }];
195 const prefix = prefixOf(commands);
196 const whole = new Uint8Array([...prefix, ...pack]);
197 const body = extra > 0 ? new Uint8Array([...whole, ...new Uint8Array(extra)]) : whole.subarray(0, whole.length + extra);
198 const r = await exchange(url, { stream: ReadableStream.from([body]), length: whole.length });
199 const report = r.error === null && r.done.tag === 'report' ? plyPush.classifyReport(r.done.lines, commands) : null;
200 observed[name] = { error: r.error?.code ?? null, report: report?.tag ?? r.done?.tag ?? null, landed: rev(bare, 'refs/heads/main') === commit, spanBytes: r.spans[0].attrs['git.request.bytes'], sent: body.length };
201 assert.equal(r.spans[0].attrs['git.request.bytes'], whole.length, `${name}: the span reports the declared length`);
202 }
203 console.log(JSON.stringify(observed));
204 // git reads a pack to its trailer: 7 bytes after it change nothing, so the push lands
205 assert.deepEqual({ ...observed.longer, spanBytes: 0 }, { error: null, report: 'statuses', landed: true, spanBytes: 0, sent: observed.longer.sent });
206 // a pack cut short of its trailer: git refuses it (unpack failed, or no report at all), and nothing lands
207 assert.equal(observed.shorter.landed, false);
208 assert.ok(observed.shorter.error !== null || observed.shorter.report === 'unpackFailed' || observed.shorter.report === 'broken', JSON.stringify(observed.shorter));
209});
211test('receive-pack refuses a command (stale old id): the report says ng, classifyReport names it; the ref is unchanged', async () => {
212 const run = compile(blocks[0].code);
213 const { bare, url } = repo();
214 const first = await run({ url, opts: { auth: { token: TOKEN }, fetchImpl: gate } });
215 const { commit, pack } = await rootCommit('second');
216 const commands = [{ ref: 'refs/heads/main', oldOid: ply.ZERO_OID, newOid: commit }];
217 const prefix = prefixOf(commands);
218 const r = await exchange(url, { stream: ReadableStream.from([prefix, pack]), length: prefix.length + pack.length });
219 assert.equal(r.error, null);
220 assert.equal(r.done.tag, 'report');
221 const report = plyPush.classifyReport(r.done.lines, commands);
222 console.log(JSON.stringify(report));
223 assert.equal(report.tag, 'statuses');
224 assert.equal(report.refs[0].ok, false);
225 assert.ok(report.refs[0].reason.length > 0);
226 assert.equal(rev(bare, 'refs/heads/main'), first.commit, 'unchanged');
227});
229test('a corrupt pack: the report is unpackFailed, the span is not an error (the exchange worked), nothing lands', async () => {
230 const { bare, url } = repo();
231 const { commit, pack } = await rootCommit('corrupt');
232 const bad = Uint8Array.from(pack); bad[20] ^= 0xff;
233 const commands = [{ ref: 'refs/heads/main', oldOid: ply.ZERO_OID, newOid: commit }];
234 const prefix = prefixOf(commands);
235 const r = await exchange(url, { stream: ReadableStream.from([prefix, bad]), length: prefix.length + bad.length });
236 assert.equal(r.error, null, String(r.error));
237 const report = r.done.tag === 'report' ? plyPush.classifyReport(r.done.lines, commands) : r.done;
238 console.log(JSON.stringify(report));
239 assert.ok(report.tag === 'unpackFailed' || report.tag === 'broken', JSON.stringify(report));
240 assert.equal(rev(bare, 'refs/heads/main'), null);
241});
243test('a server that is not git (HTML 200) is not_git; an aborted signal is aborted, and no fetch goes out live', async () => {
244 const html = async () => new Response('<html>login</html>', { status: 200, headers: { 'content-type': 'text/html' } });
245 const t = ply.createTransport('https://git.example/r.git', { fetchImpl: html });
246 const e = await (async () => { try { for await (const _ of plyTransport.frames(t.value, 'git-receive-pack', new Uint8Array(4).fill(48), t.value.tracer.start('x'))) { /* none */ } } catch (x) { return x; } return null; })();
247 assert.equal(e.code, 'not_git');
248 const { url } = repo();
249 const ac = new AbortController();
250 ac.abort();
251 const before = seen.length;
252 const r = await exchange(url, new Uint8Array(4).fill(48), { signal: ac.signal });
253 console.log(JSON.stringify({ code: r.error?.code, calls: seen.slice(before).map(x => ({ url: x.url, aborted: x.aborted })) }));
254 assert.ok(isPlyError(r.error));
255 assert.equal(r.error.code, 'aborted');
256 assert.equal(r.spans[0].status.code, 2);
257 assert.ok(seen.slice(before).every(x => x.aborted), 'every fetch it made carried the aborted signal (none went out)');
258});
Diagnostics6 git calls · 177 ms
CallStatusSizeTime0177 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=11990 git.response.cache=UPDATING git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=167 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetch2005.8 KiB167 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=5949 git.response.cache=MISS git.response.duration_ms=167 git.fetch.wants=4adfb910a31b18d9f5282f695c802685583a1594 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5949 git.pkt.payload.max=5922 git.sideband.band1.bytes=5922 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2bdca5b96cfc3
- Request
- cda0b2b6…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format