GitHubivarvong
ply
main
1// git is first-wins for a second `tree` and still fsck-clean, so a last-wins parser reads a different tree than
2// `git log --format=%T`. These objects are the ones a red-team audit used. ply rejects them on purpose.
3import { test } from 'vitest';
4import assert from 'node:assert/strict';
5import { spawnSync } from 'node:child_process';
6import { mkdtempSync, rmSync } from 'node:fs';
7import { tmpdir } from 'node:os';
8import { join } from 'node:path';
9import { te } from '../src/bytes.js';
10import { parseCommit, parseTag, parseTree, encodeCommit, ident } from '../src/objects.js';
11import { validateIdent } from '../src/validate.js';
12import { fsck, fsckProblemsOf } from './server.js';
13import { unwrap } from './unwrap.js';
15const env = {
16 ...process.env,
17 GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'commit.gpgsign', GIT_CONFIG_VALUE_0: 'false',
18 GIT_AUTHOR_NAME: 'A', GIT_AUTHOR_EMAIL: 'a@b.c', GIT_COMMITTER_NAME: 'C', GIT_COMMITTER_EMAIL: 'c@b.c',
19};
20const author = unwrap(ident(unwrap(validateIdent({ name: 'A', email: 'a@b.c', time: 1700000000 }))));
21const other = unwrap(ident(unwrap(validateIdent({ name: 'B', email: 'b@b.c', time: 1700000001 }))));
23function git(dir, args, input) {
24 const r = spawnSync('git', ['-C', dir, ...args], { input, encoding: 'utf8', env });
25 return { code: r.status ?? 1, out: `${r.stdout ?? ''}`, err: `${r.stderr ?? ''}` };
26}
28function withRepo(fn) {
29 const dir = mkdtempSync(join(tmpdir(), 'ply-diff-'));
30 try {
31 const init = git(dir, ['init', '-q', '-b', 'main']);
32 assert.equal(init.code, 0, init.err);
33 const empty = git(dir, ['hash-object', '-t', 'tree', '-w', '--stdin'], '');
34 assert.equal(empty.code, 0, empty.err);
35 return fn(dir, empty.out.trim());
36 } finally {
37 rmSync(dir, { recursive: true, force: true });
38 }
39}
41function literally(dir, type, text) {
42 const r = git(dir, ['hash-object', '-t', type, '-w', '--stdin', '--literally'], text);
43 assert.equal(r.code, 0, r.err);
44 return r.out.trim();
45}
48test('a second tree header is fsck-clean and git shows the first; ply rejects it', () => {
49 withRepo((dir, empty) => {
50 const otherTree = 'a'.repeat(40);
51 const text = `tree ${empty}\nauthor ${author}\ncommitter ${author}\ntree ${otherTree}\n\nmsg\n`;
52 const oid = literally(dir, 'commit', text);
53 const raw = git(dir, ['cat-file', '-p', oid]);
54 assert.equal(raw.code, 0);
55 assert.ok(raw.out.includes(`tree ${empty}`));
56 assert.ok(raw.out.includes(`tree ${otherTree}`));
57 assert.deepEqual(fsckProblemsOf(dir, oid), []);
58 const shown = git(dir, ['log', '-1', '--no-walk', '--format=%T', oid]);
59 assert.equal(shown.code, 0, shown.err);
60 assert.equal(shown.out.trim(), empty);
61 assert.throws(() => unwrap(parseCommit(te.encode(text))), e => e.code === 'pack_corrupt' && e.message === 'commit: duplicate tree');
62 });
63});
65test('duplicate encoding, duplicate committer, and a parent after committer', () => {
66 withRepo((dir, empty) => {
67 const enc = `tree ${empty}\nauthor ${author}\ncommitter ${author}\nencoding ISO-8859-1\nencoding UTF-8\n\nmsg\n`;
68 const encOid = literally(dir, 'commit', enc);
69 assert.deepEqual(fsckProblemsOf(dir, encOid), []);
70 const encLog = git(dir, ['log', '-1', '--no-walk', '--format=%e', encOid]);
71 assert.equal(encLog.out.trim(), 'ISO-8859-1');
72 assert.throws(() => unwrap(parseCommit(te.encode(enc))), e => e.code === 'pack_corrupt' && e.message === 'commit: duplicate encoding');
74 const com = `tree ${empty}\nauthor ${author}\ncommitter ${author}\ncommitter ${other}\n\nmsg\n`;
75 const comOid = literally(dir, 'commit', com);
76 assert.deepEqual(fsckProblemsOf(dir, comOid), []);
77 const comLog = git(dir, ['log', '-1', '--no-walk', '--format=%cn', comOid]);
78 assert.equal(comLog.out.trim(), 'B');
79 assert.throws(() => unwrap(parseCommit(te.encode(com))), e => e.code === 'pack_corrupt' && e.message === 'commit: duplicate committer');
81 const parent = 'b'.repeat(40);
82 const late = `tree ${empty}\nauthor ${author}\ncommitter ${author}\nparent ${parent}\n\nmsg\n`;
83 const lateOid = literally(dir, 'commit', late);
84 assert.deepEqual(fsckProblemsOf(dir, lateOid), []);
85 const parents = git(dir, ['log', '-1', '--no-walk', '--format=%P', lateOid]);
86 assert.equal(parents.out.trim(), '');
87 assert.throws(() => unwrap(parseCommit(te.encode(late))), e => e.code === 'pack_corrupt' && e.message === 'commit: bad header order');
88 });
89});
91test('a commit git writes round-trips, and a duplicate tagger is fsck-clean first-wins', () => {
92 withRepo((dir, empty) => {
93 const text = `tree ${empty}\nparent ${'c'.repeat(40)}\nauthor ${author}\ncommitter ${other}\nencoding UTF-8\n\nmsg\n`;
94 const oid = literally(dir, 'commit', text);
95 const log = git(dir, ['log', '-1', '--no-walk', '--date=raw', '--format=%T%x00%P%x00%an <%ae> %ad%x00%cn <%ce> %cd%x00%e', oid]);
96 assert.equal(log.code, 0, log.err);
97 const [tree, parents, who, committer, encoding] = log.out.trim().split('\0');
98 const c = unwrap(parseCommit(te.encode(text)));
99 assert.equal(c.tree, tree);
100 assert.equal(c.parents.join(' '), parents);
101 assert.equal(c.author, who);
102 assert.equal(c.committer, committer);
103 assert.equal(c.encoding, encoding);
104 assert.deepEqual([...unwrap(encodeCommit(c))], [...te.encode(text)]);
106 const tag = `object ${oid}\ntype commit\ntag v1\ntagger ${author}\ntagger ${other}\n\nrel\n`;
107 const tagOid = literally(dir, 'tag', tag);
108 assert.equal(git(dir, ['update-ref', 'refs/tags/v1', tagOid]).code, 0);
109 assert.deepEqual(fsckProblemsOf(dir, tagOid), []);
110 const shown = git(dir, ['for-each-ref', '--format=%(tagger)', 'refs/tags/v1']);
111 assert.equal(shown.out.trim(), author);
112 assert.throws(() => unwrap(parseTag(te.encode(tag))), e => e.code === 'pack_corrupt' && e.message === 'tag: duplicate tagger');
113 });
114});
116// ---- issue #34: one name twice in a tree, every cell of the product, against `git fsck --strict` ----
117// Axes (tagged unions; the test takes their product and runs every cell):
118// relation equal (`d`,`d`) | slash-prefix (`d`,`d.x`: which sorts first depends on whether `d` is a directory) |
119// case-only (`d`,`D`) | nfc-nfd (`é` as U+00E9 and as e + U+0301) | decode-twin (`d\xff`,`d\xfe`: distinct
120// bytes that a lossy decoder made one string; with decodeBytes, #43, they are two names)
121// modes blob | exec | symlink | tree | gitlink, for each of the two entries (25)
122// position adjacent | separated (a blob whose key sorts strictly between the two keys, when one exists)
123// order sorted (git's order, ties keep the first) | unsorted (that list reversed)
124// Expected (git's rules, fsck.c verify_ordered; checked against fsck below, cell by cell): fsck errors iff the two names
125// are the same bytes (duplicateEntries) or the list is out of order (treeNotSorted). ply accepts iff fsck is silent,
126// with no exception (#34's decode-twin refusal is gone since names decode losslessly, #43). When ply refuses, its
127// message is one fsck printed for that tree.
128const MODES = [['blob', '100644'], ['exec', '100755'], ['symlink', '120000'], ['tree', '40000'], ['gitlink', '160000']];
129const RELATIONS = [
130 ['equal', Buffer.from('d'), Buffer.from('d')],
131 ['slash-prefix', Buffer.from('d'), Buffer.from('d.x')],
132 ['case-only', Buffer.from('d'), Buffer.from('D')],
133 ['nfc-nfd', Buffer.from('é'), Buffer.from('é')],
134 ['decode-twin', Buffer.from([0x64, 0xff]), Buffer.from([0x64, 0xfe])],
135];
136const POSITIONS = ['adjacent', 'separated'];
137const ORDERS = ['sorted', 'unsorted'];
139const gitRaw = (dir, args, input) => {
140 const r = spawnSync('git', ['-C', dir, ...args], { input, env });
141 assert.equal(r.status, 0, `${args.join(' ')}: ${r.stderr}`);
142 return r.stdout.toString().trim();
143};
144const sortKey34 = (name, mode) => (mode === '40000' ? Buffer.concat([name, Buffer.from('/')]) : name);
145const between = (lo, hi) => {
146 const [a, b] = Buffer.compare(lo.key, hi.key) <= 0 ? [lo, hi] : [hi, lo];
147 for (const n of [a.name, b.name]) {
148 const bumped = Buffer.from(n); bumped[bumped.length - 1]++;
149 for (const m of [Buffer.concat([n, Buffer.from('!')]), Buffer.concat([n, Buffer.from('.')]), bumped]) {
150 // a third name: `d\xfe` bumped is `d\xff`, which would make the middle entry a duplicate of its own
151 if (!m.equals(a.name) && !m.equals(b.name) && Buffer.compare(a.key, m) < 0 && Buffer.compare(m, b.key) < 0) return m;
152 }
153 }
154 return null;
155};
157export function cells34() {
158 const out = [];
159 for (const [relation, nameA, nameB] of RELATIONS) for (const [ma, modeA] of MODES) for (const [mb, modeB] of MODES) for (const position of POSITIONS) for (const order of ORDERS) {
160 const A = { name: nameA, mode: modeA, key: sortKey34(nameA, modeA) }, B = { name: nameB, mode: modeB, key: sortKey34(nameB, modeB) };
161 let list = [A, B];
162 let realized = position;
163 if (position === 'separated') {
164 const m = between(A, B);
165 // no name sorts strictly between two equal keys: the cell exists, the middle entry sorts outside them
166 if (m === null) realized = 'separated-unrealizable';
167 list.push({ name: m ?? Buffer.concat([nameA, Buffer.from('!')]), mode: '100644', key: m ?? Buffer.concat([nameA, Buffer.from('!')]), middle: true });
168 }
169 list = list.map((e, i) => ({ e, i })).sort((x, y) => Buffer.compare(x.e.key, y.e.key) || x.i - y.i).map(x => x.e);
170 if (order === 'unsorted') list.reverse();
171 const sameBytes = nameA.equals(nameB);
172 const unsorted = list.some((e, i) => i > 0 && Buffer.compare(list[i - 1].key, e.key) > 0);
173 out.push({ id: `${relation}/${ma}+${mb}/${position}/${order}`, relation, modes: [ma, mb], position, realized, order, list, expectGitRejects: sameBytes || unsorted });
174 }
175 return out;
176}
178export function plyVerdict34(bytes) {
179 try { unwrap(parseTree(new Uint8Array(bytes))); return 'accept'; } catch (e) {
180 assert.equal(e.code, 'pack_corrupt', e.message);
181 if (e.message.startsWith('tree: duplicate entry ')) return 'duplicateEntries';
182 if (e.message === 'tree: not strictly sorted') return 'treeNotSorted';
183 return `other:${e.message}`;
184 }
185}
187test('#34 parseTree on every cell of relation × mode × mode × position × order agrees with git fsck --strict', () => {
188 withRepo(dir => {
189 const blob = gitRaw(dir, ['hash-object', '-w', '--stdin'], Buffer.from('x\n'));
190 const link = gitRaw(dir, ['hash-object', '-w', '--stdin'], Buffer.from('target'));
191 const sub = gitRaw(dir, ['mktree'], Buffer.from(`100644 blob ${blob}\tf\n`));
192 const gitlink = 'c0ffee'.padEnd(40, '0');
193 const oidFor = mode => ({ '100644': blob, '100755': blob, '120000': link, '40000': sub, '160000': gitlink })[mode];
194 const cells = cells34();
195 assert.equal(cells.length, RELATIONS.length * MODES.length ** 2 * POSITIONS.length * ORDERS.length);
196 for (const c of cells) {
197 c.bytes = Buffer.concat(c.list.flatMap(e => [Buffer.from(`${e.mode} `), e.name, Buffer.from([0]), Buffer.from(oidFor(e.mode), 'hex')]));
198 c.oid = gitRaw(dir, ['hash-object', '-t', 'tree', '-w', '--stdin', '--literally'], c.bytes);
199 }
200 const fsckOut = fsck(dir, ['--no-dangling']);
201 const fsckOf = new Map();
202 for (const line of fsckOut.lines) {
203 const m = /^error in tree ([0-9a-f]{40}): (\w+):/.exec(line);
204 if (m) { if (!fsckOf.has(m[1])) fsckOf.set(m[1], new Set()); fsckOf.get(m[1]).add(m[2]); }
205 }
206 const tally = new Map();
207 const bad = [];
208 for (const c of cells) {
209 const ids = fsckOf.get(c.oid) ?? new Set();
210 const ply = plyVerdict34(c.bytes);
211 const gitRejects = ids.size > 0;
212 if (gitRejects !== c.expectGitRejects) bad.push(`${c.id}: model says git ${c.expectGitRejects ? 'rejects' : 'accepts'}, fsck: [${[...ids]}]`);
213 for (const id of ids) if (id !== 'duplicateEntries' && id !== 'treeNotSorted') bad.push(`${c.id}: unexpected fsck ${id}`);
214 if (gitRejects ? !ids.has(ply) : ply !== 'accept') bad.push(`${c.id}: fsck [${[...ids]}], ply ${ply}`);
215 const k = `${c.relation} ${gitRejects ? [...ids].sort().join('+') : 'clean'} -> ${ply}`;
216 tally.set(k, (tally.get(k) ?? 0) + 1);
217 }
218 console.log(` #34 cells: ${cells.length}, separated-unrealizable: ${cells.filter(c => c.realized === 'separated-unrealizable').length}`);
219 for (const [k, n] of [...tally].sort()) console.log(` #34 ${k}: ${n}`);
220 assert.deepEqual(bad, []);
221 });
222});
Diagnostics6 git calls · 45 ms
CallStatusSizeTime045 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=5 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=11990 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit5 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=5 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=34 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetch2004.2 KiB · hit34 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=4264 git.response.cache=HIT git.response.duration_ms=34 git.fetch.wants=ef09c943f25dc26fd509da4e358e0b016d8f72e0 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4264 git.pkt.payload.max=4237 git.sideband.band1.bytes=4237 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e0f84c4ebb332c
- Request
- dba2b8d0…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format