GitHubivarvong

ply

main
ply6 git calls271 ms26 KiB3 cachedprotocol v2
155 lines9.1 KiBRaw
1// The timestamp of an author, committer or tagger line is the remote's text, bounded only by maxObjectSize. Deciding
2// whether it overflows git's range must not cost more than a benign object of the same size: the run of digits is
3// judged by its length before any of it is converted (objects/ident-timestamp-bigint-cpu).
4// The verdicts on the boundary values are git's: `git fsck --strict` over a repository holding the hand-built object.
5import { test } from 'vitest';
6import assert from 'node:assert/strict';
7import { spawnSync } from 'node:child_process';
8import { mkdtempSync, rmSync } from 'node:fs';
9import { tmpdir } from 'node:os';
10import { join } from 'node:path';
11import { parseCommit, parseTag, fsckIdent } from '../src/objects.js';
13const TREE = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'; // git's empty tree
14const latin1 = s => new Uint8Array(Buffer.from(s, 'latin1'));
15// Hand-built from the object formats, not ply's encoders.
16const commitText = (time, message = 'm\n') => `tree ${TREE}\nauthor A <a@b> ${time} +0000\ncommitter A <a@b> 1700000000 +0000\n\n${message}`;
17const tagText = (time, message = 'm\n') => `object ${TREE}\ntype tree\ntag v1\ntagger A <a@b> ${time} +0000\n\n${message}`;
19// The work counter: every digit handed to BigInt while fn runs. Converting a decimal string to a BigInt is the only
20// superlinear step an ident parse can take, so this is the cost the remote controls. (If the parser ever stops using
21// BigInt, the benign assertion below fails and this counter is replaced; it does not silently pass.)
22function digitsConverted(fn) {
23 const real = globalThis.BigInt;
24 let digits = 0;
25 globalThis.BigInt = new Proxy(real, {
26 apply(target, self, args) {
27 if (typeof args[0] === 'string') digits += args[0].length;
28 return Reflect.apply(target, self, args);
29 },
30 });
31 try { return { value: fn(), digits }; } finally { globalThis.BigInt = real; }
34// 2^63 - 1 is 19 digits, so no verdict needs more than 19 digits of any one timestamp.
35const MOST_DIGITS_ONE_IDENT_NEEDS = 19;
37test('a huge timestamp costs no more conversion work than a benign object of the same size, and is still badDateOverflow', () => {
38 for (const [parse, text, idents] of [[parseCommit, commitText, 2], [parseTag, tagText, 1]]) {
39 for (const n of [100_000, 200_000, 400_000]) {
40 const hostileBytes = latin1(text('9'.repeat(n)));
41 // the same size, the bulk in the message; its idents carry ordinary 10-digit times
42 const benignBytes = latin1(text('1700000000', `${'9'.repeat(n - 10)}m\n`));
43 assert.equal(benignBytes.length, hostileBytes.length, 'same-size objects');
44 const benign = digitsConverted(() => parse(benignBytes));
45 assert.equal(benign.value.ok, true, 'benign object parses');
46 // the counter is live: each ident of the benign object has a 10-digit time
47 assert.equal(benign.digits, 10 * idents, `${parse.name} benign n=${n}: digits converted`);
48 const hostile = digitsConverted(() => parse(hostileBytes));
49 assert.equal(hostile.value.ok, false);
50 assert.equal(hostile.value.error.code, 'pack_corrupt');
51 assert.equal(hostile.value.error.msgId, 'badDateOverflow');
52 assert.ok(hostile.digits <= MOST_DIGITS_ONE_IDENT_NEEDS * idents,
53 `${parse.name} n=${n}: ${hostile.digits} digits converted for the hostile object, ${benign.digits} for the same-size benign one (ratio ${(hostile.digits / benign.digits).toFixed(1)}; bound ${MOST_DIGITS_ONE_IDENT_NEEDS * idents})`);
54 }
55 }
56});
58// The second work counter: every character the ident parser classifies as a digit. The digit run was still walked to
59// its end one classification a digit after BigInt stopped seeing it: linear, but about 200 times the CPU of a same-size
60// benign object (the reviewer's exp6-cputime.mjs on 91f0e71: 64 ms against 0.3 ms at 4M digits).
61// The expected bound is git's rule, not ply's code: with no leading zero a 20th digit is already past 2^63 - 1, and
62// fsck_ident reports the overflow before it reads what follows the digits, so 20 digits of one timestamp decide it.
63// The count goes through the classifier fsckIdent takes as a parameter (a RegExp; parseCommit and parseTag pass the
64// module's own), so nothing global is patched (#276) and the gate c recorder's own regex calls are never counted.
65class CountingDigit extends RegExp {
66 constructor() { super('^[0-9]$', 'u'); this.tests = 0; }
67 test(s) { this.tests++; return super.test(s); }
69function charsClassified(text, at, kind, role) {
70 const digit = new CountingDigit();
71 return { value: fsckIdent(text, at, kind, role, digit), tests: digit.tests };
73const AT = { commit: `tree ${TREE}\nauthor `.length, tag: `object ${TREE}\ntype tree\ntag v1\ntagger `.length };
75test('a huge timestamp is not walked to its end: the characters classified do not grow with the digit run', () => {
76 for (const [kind, role, text] of [['commit', 'author', commitText], ['tag', 'tagger', tagText]]) {
77 const counts = [];
78 for (const n of [100_000, 200_000, 400_000]) {
79 const benign = charsClassified(text('1700000000', `${'9'.repeat(n - 10)}m\n`), AT[kind], kind, role);
80 assert.equal(benign.value.ok, true, 'benign ident parses');
81 // the counter is live, by hand: 1 for the first digit, 11 for the 10-digit run (its 10 digits and the space that
82 // ends it), 4 for the time zone digits
83 assert.equal(benign.tests, 1 + 11 + 4, `${kind} benign n=${n}: characters classified`);
84 const hostile = charsClassified(text('9'.repeat(n)), AT[kind], kind, role);
85 assert.equal(hostile.value.ok, false);
86 assert.equal(hostile.value.error.msgId, 'badDateOverflow');
87 // by hand from git's rule: 1 for the first digit, then the run is read to its 20th digit and no further
88 assert.equal(hostile.tests, 1 + 20, `${kind} n=${n}: characters classified for the hostile ident`);
89 counts.push(hostile.tests);
90 }
91 assert.deepEqual(counts, [counts[0], counts[0], counts[0]], `${kind}: the same work at 100k, 200k and 400k digits`);
92 }
93});
95// parseCommit and parseTag still refuse the hostile object (the whole-object verdict, independent of the counter)
96test('parseCommit and parseTag refuse a huge timestamp as badDateOverflow', () => {
97 for (const [parse, text] of [[parseCommit, commitText], [parseTag, tagText]]) {
98 const r = parse(latin1(text('9'.repeat(100_000))));
99 assert.equal(r.ok, false);
100 assert.equal(r.error.msgId, 'badDateOverflow');
101 }
102});
104function git(dir, args, input) {
105 const r = spawnSync('git', ['-C', dir, ...args], { input, maxBuffer: 1 << 28 });
106 return { code: r.status ?? 1, out: `${r.stdout ?? ''}`, err: `${r.stderr ?? ''}` };
108// git's verdict on one object: fsck --strict over a fresh repository holding it and the empty tree it points at.
109function fsck(type, text) {
110 const dir = mkdtempSync(join(tmpdir(), 'ply-ident-time-'));
111 try {
112 assert.equal(git(dir, ['init', '-q', '-b', 'main']).code, 0);
113 assert.equal(git(dir, ['hash-object', '-t', 'tree', '-w', '--stdin'], '').out.trim(), TREE);
114 const w = git(dir, ['hash-object', '-t', type, '-w', '--stdin', '--literally'], Buffer.from(text, 'latin1'));
115 assert.equal(w.code, 0, w.err);
116 const r = git(dir, ['fsck', '--strict']);
117 return { accepted: r.code === 0, overflow: /badDateOverflow/u.test(r.err + r.out), text: (r.err + r.out).trim() };
118 } finally {
119 rmSync(dir, { recursive: true, force: true });
120 }
123// Around 2^63 - 1 (9223372036854775807) and the 19 / 20-digit edge the length rule decides on. Git says which overflow.
124const BOUNDARY = [
125 '0', '1700000000',
126 '999999999999999999', // 18 digits
127 '1000000000000000000', // the least 19-digit value
128 '9223372036854775806', '9223372036854775807', '9223372036854775808', // 2^63 - 2, 2^63 - 1, 2^63
129 '9999999999999999999', // the greatest 19-digit value
130 '10000000000000000000', // the least 20-digit value
131 '18446744073709551615', '18446744073709551616', // 2^64 - 1, 2^64
132 '99999999999999999999', // the greatest 20-digit value
133 '100000000000000000000', // 21 digits
134 '9'.repeat(40),
135];
137test('the 19 / 20-digit boundary timestamps get git fsck\'s verdict from parseCommit and parseTag', () => {
138 const disagreements = [];
139 const verdicts = { accepted: 0, overflow: 0 };
140 for (const [type, parse, text] of [['commit', parseCommit, commitText], ['tag', parseTag, tagText]]) {
141 for (const time of BOUNDARY) {
142 const theirs = fsck(type, text(time));
143 // every refusal in these cells must be the overflow one: anything else means the cell is not testing the date
144 assert.equal(theirs.accepted || theirs.overflow, true, `${type} ${time}: git refused for another reason: ${theirs.text}`);
145 verdicts[theirs.accepted ? 'accepted' : 'overflow']++;
146 const ours = parse(latin1(text(time)));
147 const oursVerdict = ours.ok ? 'accepted' : `${ours.error.code} ${ours.error.msgId}`;
148 const gitVerdict = theirs.accepted ? 'accepted' : 'pack_corrupt badDateOverflow';
149 if (oursVerdict !== gitVerdict) disagreements.push(`${type} ${time} (${time.length} digits): git ${gitVerdict}, ply ${oursVerdict}`);
150 }
151 }
152 assert.deepEqual(disagreements, []);
153 // the oracle saw both sides of the boundary
154 assert.ok(verdicts.accepted >= 8 && verdicts.overflow >= 8, JSON.stringify(verdicts));
155});
Diagnostics6 git calls · 271 ms
CallStatusSizeTime0271 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB109 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=101
git.response.bytes=11990
git.response.cache=EXPIRED
git.response.duration_ms=109
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2003.4 KiB155 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=155
git.response.bytes=3500
git.response.cache=MISS
git.response.duration_ms=155
git.fetch.wants=edf7f6da8990dba0191f83a110a015a849992050
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3500
git.pkt.payload.max=3473
git.sideband.band1.bytes=3473
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2b1099f67e6b0
Request
3b938bcf…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format