main
1// Remote-written text in errors (SAFETY.md "Remote text"): a server chooses the bytes of an ERR line, a band-3 message,
2// an HTTP error body, a refused protocol line. None of them may reach an error message with a character that drives a
3// terminal or forges a control sequence in a log. Expected values here are derived by hand from the input (each escape
4// is the code point in hex, four digits in the BMP), and the "nothing unprintable" property is stated with a Unicode-category
5// regex written in this file.
6import { test } from 'vitest';
7import assert from 'node:assert/strict';
8import { readFileSync, readdirSync } from 'node:fs';
9import http from 'node:http';
10import { remoteQuoted, remoteText } from '../src/errors.js';
11import { v2Init, v2Step, v2End } from '../src/fetch.js';
12import { parseCapabilitiesV2 } from '../src/caps.js';
13import { parseLsRefLine } from '../src/lsrefs.js';
14import { outcomeResult } from '../src/push.js';
15import { listRefs, readFile } from '../src/index.js';
17const te = new TextEncoder();
18// what a terminal or log acts on: controls (C0, DEL, C1) except tab and newline, format characters, separators, lone surrogates
19const DRIVES = /(?![\t\n])[\p{Cc}\p{Cf}\p{Cs}\p{Zl}\p{Zp}]/u;
21test('remoteText: ESC sequences, CR, NUL, C1, DEL and bidi overrides are written as \\uXXXX', () => {
22 // ESC = 0x1b, BEL = 0x07, CR = 0x0d, NUL = 0x00, CSI = 0x9b, DEL = 0x7f, RLO = 0x202e
23 assert.equal(remoteText('\x1b]0;owned\x07\x1b[2J ok\r\x1b[K'), '\\u001b]0;owned\\u0007\\u001b[2J ok\\u000d\\u001b[K');
24 assert.equal(remoteText('a\0b'), 'a\\u0000b');
25 assert.equal(remoteText('x\u009b31m\x7f'), 'x\\u009b31m\\u007f\\u202e');
26 // printable text, tab and newline are unchanged
27 assert.equal(remoteText('denied to ivar\n\tsee https://example.com/é — 日本'), 'denied to ivar\n\tsee https://example.com/é — 日本');
28 // quoted: JSON's escapes for quote, backslash and newline, then DEL, C1, a bidi override and U+2028
29 assert.equal(remoteQuoted('a"b\\c\n\x7f\u009b
', 60), '"a\\"b\\\\c\\n\\u007f\\u009b\\u202e\\u2028"');
30});
32test('remoteText: a 10 MB line is cut to the bound before any work, and a cut surrogate pair is escaped', () => {
33 const big = '\x1b'.repeat(10 * 1024 * 1024);
34 assert.equal(remoteText(big, 60), '\\u001b'.repeat(60));
35 assert.equal(remoteQuoted(big, 60), `"${'\\u001b'.repeat(60)}"`);
36 assert.equal(remoteText('a'.repeat(3000)).length, 1024); // the default bound
37 // 59 units then U+1F600 (d83d de00): the cut at 60 leaves the lone high surrogate d83d
38 assert.equal(remoteText('a'.repeat(59) + '\u{1f600}', 60), `${'a'.repeat(59)}\\ud83d`);
39 assert.equal(remoteText('a'.repeat(58) + '\u{1f600}', 60), `${'a'.repeat(58)}\u{1f600}`);
40});
42test('remoteText: invalid UTF-8 decodes to U+FFFD and nothing unprintable is left', () => {
43 const decoded = new TextDecoder().decode(Uint8Array.from([0x41, 0xff, 0xfe, 0xc3, 0x28, 0x1b, 0x42]));
44 assert.equal(remoteText(decoded), 'A���(\\u001bB');
45 for (let cp = 0; cp < 0x3000; cp++) {
46 assert.equal(DRIVES.test(remoteText(`x${String.fromCodePoint(cp)}y`)), false, `U+${cp.toString(16)}`);
47 assert.equal(DRIVES.test(remoteQuoted(`x${String.fromCodePoint(cp)}y`)), false, `quoted U+${cp.toString(16)}`);
48 }
49 // lone surrogates, and U+E0001 (a tag character, above the BMP)
50 assert.equal(remoteText('\ud800 \udfff \u{e0001}'), '\\ud800 \\udfff \\u{e0001}');
51 assert.equal(DRIVES.test(remoteQuoted('\ud800\x1b\u{e0001}')), false);
52});
54// the v2 fetch answer, one frame at a time (the frames are written here, not by ply)
55const step = (lines, done) => {
56 let st = v2Init();
57 for (const l of [...lines, null]) { const r = v2Step(st, l === null ? { kind: 'flush' } : { kind: 'data', payload: te.encode(l) }); if (!r.ok) return r; st = r.value.state; }
58 return v2End(st, done);
59};
61test('fetch: a refused acknowledgment line is quoted cut to 60 units and escaped; the ready message is exact', () => {
62 const junk = step(['acknowledgments\n', 'x\x1b[2J\u009b31m\x7f\n'], false);
63 assert.deepEqual([junk.ok, junk.error.code, junk.error.message], [false, 'protocol', 'v2: unexpected acknowledgment line "x\\u001b[2J\\u009b31m\\u007f\\u202e"']);
64 // boundary: 60 units are shown whole, the 61st is cut
65 const at60 = step(['acknowledgments\n', `${'k'.repeat(60)}\n`], false);
66 assert.equal(at60.error.message, `v2: unexpected acknowledgment line "${'k'.repeat(60)}"`);
67 const at61 = step(['acknowledgments\n', `${'k'.repeat(60)}Z\n`], false);
68 assert.equal(at61.error.message, `v2: unexpected acknowledgment line "${'k'.repeat(60)}"`);
69 const ready = step(['acknowledgments\n', 'ready\n'], false);
70 assert.deepEqual([ready.ok, ready.error.code, ready.error.message], [false, 'protocol', 'v2: the server said ready and sent no packfile section']);
71});
73test('fetch: an ERR line reaches remote_error escaped', () => {
74 const r = step(['ERR \x1b]0;owned-title\x07\x1b[2J\x1b[1;1H\x1b[32mEverything up-to-date\x1b[0m\r\x1b[K\n'], true);
75 assert.deepEqual([r.ok, r.error.code], [false, 'remote_error']);
76 assert.equal(r.error.message, 'remote: \\u001b]0;owned-title\\u0007\\u001b[2J\\u001b[1;1H\\u001b[32mEverything up-to-date\\u001b[0m\\u000d\\u001b[K');
77 assert.equal(DRIVES.test(r.error.message), false);
78});
80// v2End names the sections of an answer that has no packfile section. A section header is a line the server chooses:
81// each name is shown cut to 60 units and escaped, and at most 8 names are shown. Expected values are written out by hand.
82test('fetch: the section names in "no packfile section" are escaped, cut to 60 units each, and at most 8 are shown', () => {
83 // ESC ] 0 ; owned BEL, ESC [ 2 J, C1 CSI (U+009B), RLO (U+202E)
84 const hostile = step(['\x1b]0;owned\x07\x1b[2J\u009b31m\n'], true);
85 assert.deepEqual([hostile.ok, hostile.error.code], [false, 'protocol']);
86 assert.equal(hostile.error.message, 'v2: no packfile section (sections: \\u001b]0;owned\\u0007\\u001b[2J\\u009b31m\\u202e)');
87 assert.equal(DRIVES.test(hostile.error.message), false);
88 // names git writes are shown as they were, and no section at all is still "none"
89 assert.equal(step(['acknowledgments\n', 'NAK\n'], true).error.message, 'v2: no packfile section (sections: acknowledgments)');
90 assert.equal(step(['shallow-info\n'], false).error.message, 'v2: no packfile section (sections: shallow-info)');
91 assert.equal(step([], true).error.message, 'v2: no packfile section (sections: none)');
92 // one header of 65000 units: 60 are shown. 61 units: the 61st is cut.
93 assert.equal(step([`${'a'.repeat(65000)}\n`], true).error.message, `v2: no packfile section (sections: ${'a'.repeat(60)})`);
94 assert.equal(step([`${'k'.repeat(60)}Z\n`], true).error.message, `v2: no packfile section (sections: ${'k'.repeat(60)})`);
95 // 12 sections (a delim frame between them starts the next header): the first 8 are shown. 8 names: all 8.
96 const many = n => { let st = v2Init(); for (let i = 0; i < n; i++) { st = v2Step(st, data(`s${i}\n`)).value.state; st = v2Step(st, { kind: 'delim' }).value.state; } return v2End(v2Step(st, { kind: 'flush' }).value.state, true); };
97 assert.equal(many(12).error.message, 'v2: no packfile section (sections: s0,s1,s2,s3,s4,s5,s6,s7)');
98 assert.equal(many(8).error.message, 'v2: no packfile section (sections: s0,s1,s2,s3,s4,s5,s6,s7)');
99 // the whole message is bounded whatever the server sends: 8 names of 60 escaped units (6 characters each) and 7 commas
100 const worst = (() => { let st = v2Init(); for (let i = 0; i < 12; i++) { st = v2Step(st, data(`${'\x1b'.repeat(65000)}\n`)).value.state; st = v2Step(st, { kind: 'delim' }).value.state; } return v2End(v2Step(st, { kind: 'flush' }).value.state, true); })();
101 assert.equal(worst.error.message.length, 'v2: no packfile section (sections: )'.length + 8 * 60 * 6 + 7);
102});
104// A credential the server echoes is scrubbed before the text is cut (SAFETY.md I43): a cut that falls inside the
105// credential must leave none of it. The token is 40 capital letters; the padding is dots and every other character of
106// the expected message is lower case, a digit, a bracket, a colon or a space, so "no character of the token is in the
107// message" is checked without ply's scrubber. The expected message is derived by hand: the token is replaced by the
108// scrubber's marker "[redacted]", then 1024 units of the text are kept.
109const TOKEN = 'ZQXLWJVKMNPTYHBGFSUCZQXLWJVKMNPTYHBGFSUC';
110const CUTS = [1024 - 39, 1024 - 20, 1024 - 11, 1024 - 5, 1024 - 1]; // padding lengths that put the cut inside the token
111const echoed = pad => `${'.'.repeat(pad)}${TOKEN} denied`;
112const kept = pad => `${'.'.repeat(pad)}[redacted] denied`.slice(0, 1024);
113const leaked = message => [...new Set(TOKEN)].filter(c => message.includes(c));
114const handScrub = s => s.replaceAll(TOKEN, '[redacted]');
115const data = s => ({ kind: 'data', payload: te.encode(s) });
117test('a credential across the 1024-unit cut: the step functions scrub first (caps, ls-refs, fetch ERR, fetch band 3, push band 3)', () => {
118 for (const pad of CUTS) {
119 const sites = {
120 caps: parseCapabilitiesV2([data(`ERR ${echoed(pad)}`)], handScrub),
121 lsRefs: parseLsRefLine(`ERR ${echoed(pad)}`, handScrub),
122 fetchErr: v2Step(v2Init(), data(`ERR ${echoed(pad)}\n`), handScrub),
123 fetchBand3: v2Step(v2Step(v2Init(), data('packfile\n')).value.state, data(`\x03${echoed(pad)}\n`), handScrub),
124 };
125 for (const [site, r] of Object.entries(sites)) {
126 assert.deepEqual([site, r.ok, r.error.code, r.error.message], [site, false, 'remote_error', `remote: ${kept(pad)}`]);
127 assert.deepEqual(leaked(r.error.message), [], `${site} pad=${pad}`);
128 }
129 const pushed = outcomeResult({ tag: 'ambiguous', why: 'band 3' }, { report: null, hookOutput: '', remote: echoed(pad), commands: [] }, handScrub);
130 assert.deepEqual([pushed.ok, pushed.error.code, pushed.error.message], [false, 'ambiguous', `push outcome unknown: remote: ${kept(pad)}: band 3`]);
131 assert.deepEqual(leaked(pushed.error.message.replace(/: band 3$/u, '')), [], `push pad=${pad}`);
132 }
133});
135test('a scrub argument that is not a function is ignored: the step returns its coded error and does not throw', () => {
136 for (const s of [null, undefined, 5, 'x', {}, true]) {
137 const r = v2Step(v2Init(), data('ERR denied\n'), s);
138 assert.deepEqual([r.ok, r.error.code, r.error.message], [false, 'remote_error', 'remote: denied']);
139 assert.equal(parseLsRefLine('ERR denied', s).error.message, 'remote: denied');
140 assert.equal(parseCapabilitiesV2([data('ERR denied\n')], s).error.message, 'remote: denied');
141 }
142});
144// The same through the exchanges, with ply's own scrubber (auth.token): a server written here answers ls-refs or fetch
145// with the token inside an ERR line or a band-3 message.
146test('a credential across the 1024-unit cut: listRefs (ERR) and readFile (fetch ERR, fetch band 3) carry none of it', async () => {
147 const pkt = s => { const b = Buffer.from(s, 'latin1'); return Buffer.concat([Buffer.from((b.length + 4).toString(16).padStart(4, '0')), b]); };
148 const FLUSH = Buffer.from('0000');
149 const advert = Buffer.concat([pkt('version 2\n'), pkt('agent=git/2.50.0\n'), pkt('ls-refs=unborn\n'), pkt('fetch=shallow wait-for-done filter\n'), pkt('object-format=sha1\n'), FLUSH]);
150 let answer = () => FLUSH;
151 const server = http.createServer(async (req, res) => {
152 const chunks = []; for await (const c of req) chunks.push(c);
153 if (req.method === 'GET') { res.writeHead(200, { 'content-type': 'application/x-git-upload-pack-advertisement' }); res.end(advert); return; }
154 res.writeHead(200, { 'content-type': 'application/x-git-upload-pack-result' });
155 res.end(answer(Buffer.concat(chunks).toString('latin1')));
156 });
157 await new Promise(r => server.listen(0, '127.0.0.1', r));
158 const url = `http://127.0.0.1:${server.address().port}/repo.git`;
159 const listing = Buffer.concat([pkt(`${'1'.repeat(40)} refs/heads/main\n`), FLUSH]);
160 const lsRefs = body => body.includes('command=ls-refs');
161 try {
162 for (const pad of CUTS) {
163 const cases = {
164 lsRefsErr: [() => pkt(`ERR ${echoed(pad)}\n`), () => listRefs(url, {}, { auth: { token: TOKEN } })],
165 fetchErr: [body => (lsRefs(body) ? listing : pkt(`ERR ${echoed(pad)}\n`)), () => readFile(url, { ref: 'main', path: 'a' }, { auth: { token: TOKEN } })],
166 fetchBand3: [body => (lsRefs(body) ? listing : Buffer.concat([pkt('packfile\n'), pkt(`\x03${echoed(pad)}\n`)])), () => readFile(url, { ref: 'main', path: 'a' }, { auth: { token: TOKEN } })],
167 };
168 for (const [name, [serve, call]] of Object.entries(cases)) {
169 answer = serve;
170 const r = await call();
171 assert.deepEqual([name, pad, r.ok, r.error.code, r.error.message], [name, pad, false, 'remote_error', `remote: ${kept(pad)}`]);
172 assert.deepEqual(leaked(r.error.message), [], `${name} pad=${pad}`);
173 }
174 }
175 } finally { server.closeAllConnections(); await new Promise(r => server.close(r)); }
176});
178// The inventory: every place in src/ that puts remote-written text into an error message or a span attribute, by file
179// and number of remoteText( / remoteQuoted( calls. A new site is added here in the same change; a site that quotes
180// remote text in one of the old shapes is caught by the patterns below.
181const SITES = { 'caps.ts': 2, 'fetch.ts': 4, 'http.ts': 2, 'lsrefs.ts': 4, 'negotiate.ts': 1, 'push.ts': 3, 'receivecaps.ts': 1 };
182// a message that interpolates the section names a server sent (v2End) with no helper on the line
183const RAW_SECTIONS = line => /\$\{[^`]*\bsections\b/u.test(line) && !/remote(?:Text|Quoted)\(/u.test(line);
184test('inventory: the raw-shape patterns see a section list interpolated without the helper', () => {
185 // v2End's line as it was written before it was routed, and as it is written now
186 assert.equal(RAW_SECTIONS(" if (done || !acks) return proto(`v2: no packfile section (sections: ${st.sections.join(',') || 'none'})`);"), true);
187 assert.equal(RAW_SECTIONS(" if (done || !acks) return proto(`v2: no packfile section (sections: ${st.sections.slice(0, SECTIONS_SHOWN).map(s => remoteText(s, SECTION_SHOWN)).join(',') || 'none'})`);"), false);
188});
189test('inventory: remote text reaches an error only through remoteText / remoteQuoted', () => {
190 const dir = new URL('../src/', import.meta.url);
191 const found = {};
192 const raw = [];
193 for (const name of readdirSync(dir).filter(n => n.endsWith('.ts')).sort()) {
194 if (name === 'errors.ts') continue; // the helpers themselves
195 const lines = readFileSync(new URL(name, dir), 'utf8').split('\n');
196 lines.forEach((line, i) => {
197 if (line.startsWith('import ')) return;
198 const calls = line.split(/remote(?:Text|Quoted)\(/u).length - 1;
199 if (calls > 0) found[name] = (found[name] ?? 0) + calls;
200 // the old shapes: `remote: ${…}` and a JSON-quoted remote line, without the helper
201 if (/`remote: \$\{(?!remoteText\()/u.test(line) || /JSON\.stringify\((?:(?:line|first|head|a)\.slice|first\)|ct\))/u.test(line) || RAW_SECTIONS(line)) raw.push(`${name}:${i + 1}`);
202 });
203 }
204 assert.deepEqual(raw, []);
205 assert.deepEqual(found, SITES);
206});
Diagnostics4 git calls · 249 ms
CallStatusSizeTime0249 ms
discover200152 B7 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=4 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=239 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2005.3 KiB239 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=5397 git.response.cache=MISS git.response.duration_ms=239 git.fetch.wants=dd1edddbaf5d032cfa16f74d7440990f46ece6ac git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5397 git.pkt.payload.max=5376 git.sideband.band1.bytes=5375 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2c64f0b724afc
- Request
- 25624f1f…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format