GitHubivarvong

ply

main
ply6 git calls161 ms27 KiB3 cachedprotocol v2
163 lines11 KiBRaw
1// push() sends each object once, whatever list a producer hands it. A pack that holds one object twice is refused
2// by a strict receiver. Measured with git 2.50.0 on the 379-byte pack commitChanges built for "the same new file
3// under two twins" before trees were deduplicated (the same new tree for a/ and b/, packed twice):
4// git index-pack --strict --stdin : fatal: The same object deaee133500b7fbecb042fd6c74b5f5cb61b0a9b appears twice in the pack
5// git unpack-objects --strict : fatal: object deaee133500b7fbecb042fd6c74b5f5cb61b0a9b tried to add buffer twice!
6// (both accept it without --strict), which receive-pack reports as "unpacker error".
7// Judge: git. The objects are git's own (a commit made by git, read back with cat-file); the server decides whether
8// the pack is acceptable, and the ref must then be the commit git made.
9import { test, vi, beforeAll as before, afterAll as after } from 'vitest';
10import assert from 'node:assert/strict';
11import { join } from 'node:path';
12import { mkdirSync, writeFileSync } from 'node:fs';
13import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, fsckProblems, commitFiles, initRepo } from './server.js';
14import { GitHttp } from '../src/client.js';
15import { Tracer } from '../src/otel.js';
16import { TYPE } from '../src/objects.js';
17import { makeDelta } from '../src/pack.js';
18import * as pushEntry from '../src/push.js';
19import { Sha1 } from '../src/sha1.js';
21let root, cleanup, srv, n = 0;
22before(async () => { ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root); });
23after(async () => { await srv.close(); cleanup(); });
25const CONFIGS = [
26 ['strict-unpack', []], // receive.fsckObjects=true, unpack-objects --strict
27 ['index-pack', [['receive.unpackLimit', '1']]], // receive.fsckObjects=true, index-pack --strict
28];
30// A repository with one seed commit on the server and a second commit (by git) only in the work tree.
31function fixture(cfg) {
32 const name = `once${n++}`;
33 const bare = makeBareRepo(root, name);
34 for (const [k, v] of cfg) git(bare, ['config', k, v]);
35 const work = join(root, `w${name}`); mkdirSync(work); initRepo(work);
36 const head = commitFiles(work, bare, { 'a/f.txt': 'one\n', 'b/f.txt': 'one\n' }, 'seed');
37 writeFileSync(join(work, 'a/f.txt'), 'two\n'); writeFileSync(join(work, 'b/f.txt'), 'two\n'); writeFileSync(join(work, 'c.txt'), 'c\n');
38 git(work, ['add', '-A']); git(work, ['commit', '-q', '-m', 'next']);
39 const next = git(work, ['rev-parse', 'HEAD']).trim();
40 const raw = (type, rev) => ({ type: TYPE[type.toUpperCase()], data: new Uint8Array(gitBuf(work, ['cat-file', type, rev])) });
41 // a/ and b/ are one new tree, their f.txt one new blob: what a producer walking both paths collects twice
42 const objects = [raw('commit', 'HEAD'), raw('tree', 'HEAD^{tree}'), raw('tree', 'HEAD:a'), raw('blob', 'HEAD:a/f.txt'), raw('blob', 'HEAD:c.txt')];
43 return { name, bare, head, next, objects, work, raw };
46for (const [config, cfg] of CONFIGS) {
47 test(`push: a list that names objects twice is one object each on the wire [${config}]`, async () => {
48 const f = fixture(cfg);
49 const [commit, rootTree, sub, blob, other] = f.objects;
50 // every object twice, the repeats as separate copies of the bytes (not the same array) and out of order
51 const copy = o => ({ type: o.type, data: o.data.slice() });
52 const objects = [commit, rootTree, sub, copy(sub), blob, other, copy(blob), copy(rootTree), copy(commit), copy(other)];
53 const spans = [];
54 const c = GitHttp.create(`${srv.url}/${f.name}.git`, { tracer: new Tracer({ onEnd: s => spans.push(s) }) });
55 assert.equal(c.ok, true);
56 const r = await c.value.push({ ref: 'refs/heads/main', oldOid: f.head, newOid: f.next, objects });
57 assert.equal(r.ok ? 'ok' : `${r.error.code}: ${r.error.message}`, 'ok');
58 assert.equal(r.value.entries.length, 5); // five distinct objects, by the fixture
59 // what README says is counted: the objects sent (5), not the list's length (10)
60 const rp = spans.find(s => s.name === 'git.receive-pack');
61 assert.equal(rp.attrs['git.push.pack.objects'], 5);
62 assert.equal(rp.events.filter(e => e.name === 'git.push.object').length, 5);
63 assert.equal(git(f.bare, ['rev-parse', 'refs/heads/main']).trim(), f.next);
64 assert.deepEqual(fsckProblems(f.bare), []);
65 });
67 test(`push: an ofs delta whose base is a repeated object still names its base [${config}]`, async () => {
68 const f = fixture(cfg);
69 const [commit, rootTree, sub, blob, other] = f.objects;
70 // c.txt as a delta against the a/f.txt blob, which sits after a dropped repeat: its ofs index (4) must follow
71 const delta = { type: TYPE.BLOB, data: makeDelta(blob.data, other.data).delta, base: { tag: 'ofs', index: 4 } };
72 const objects = [commit, rootTree, sub, { type: sub.type, data: sub.data.slice() }, blob, delta];
73 const c = GitHttp.create(`${srv.url}/${f.name}.git`, { tracer: new Tracer() });
74 assert.equal(c.ok, true);
75 const r = await c.value.push({ ref: 'refs/heads/main', oldOid: f.head, newOid: f.next, objects });
76 assert.equal(r.ok ? 'ok' : `${r.error.code}: ${r.error.message}`, 'ok');
77 assert.equal(git(f.bare, ['rev-parse', 'refs/heads/main']).trim(), f.next);
78 assert.equal(git(f.bare, ['cat-file', 'blob', 'refs/heads/main:c.txt']), 'c\n');
79 assert.deepEqual(fsckProblems(f.bare), []);
80 });
83// package.json maps './push' to this module, so each of its exports is public API ('ply/push'). The guard is push()'s
84// own: main (ec337c5) exports no such name, and the owner's decision is that none is added.
85// push() returns a Result and never rejects. A JavaScript caller's list with an entry that is not an object is refused
86// on main (ec337c5) as Result err 'protocol' before any request, for each of these lists (measured: attack5.mjs of the
87// second review, run against main). The guard reads every entry, so it runs where main first read one.
88test('push: an objects list with a null or undefined entry is a protocol error, never a rejection', async () => {
89 const blob = byte => ({ type: TYPE.BLOB, data: Uint8Array.of(byte) });
90 let calls = 0;
91 const fetchImpl = async () => { calls++; throw new Error('the network must not be reached'); };
92 for (const objects of [[null, blob(1)], [blob(1), blob(2), null], [undefined, blob(1)]]) {
93 const c = GitHttp.create('http://127.0.0.1:9/x.git', { tracer: new Tracer(), fetchImpl });
94 assert.equal(c.ok, true);
95 // a rejection fails here: the await throws
96 const r = await c.value.push({ ref: 'refs/heads/x', oldOid: '0'.repeat(40), newOid: 'a'.repeat(40), objects });
97 assert.equal(r.ok ? 'ok' : r.error.code, 'protocol');
98 // planning refuses nothing main did not: it returns a Result for the same list, and does not throw
99 assert.equal(pushEntry.planPush({ ref: 'refs/heads/x', oldOid: '0'.repeat(40), newOid: 'a'.repeat(40), objects }, null).ok, true);
100 }
101 assert.equal(calls, 0);
102});
104// A list with no repeated object costs no object hash. A pack holds no object id (gitformat-pack: entries, then one
105// SHA-1 trailer over the pack), so main hashes no object to push; an object id is SHA-1 over "<type> <size>\0" and the
106// bytes (git hash-object), which makes each object hash visible as a SHA-1 update of exactly that header. Counted,
107// not timed: 24 blobs of one length (8 bytes, all distinct) beside git's commit and tree.
108test('push: a list with no repeated object hashes no object, though every blob has the same length', async () => {
109 const name = `once${n++}`;
110 const bare = makeBareRepo(root, name);
111 const work = join(root, `w${name}`); mkdirSync(work); initRepo(work);
112 const head = commitFiles(work, bare, { 'seed.txt': 'seed\n' }, 'seed');
113 const files = Array.from({ length: 24 }, (_, i) => `f${String(i).padStart(2, '0')}.txt`);
114 for (const f of files) writeFileSync(join(work, f), `${f.slice(0, 3)}:one\n`);
115 git(work, ['add', '-A']); git(work, ['commit', '-q', '-m', 'next']);
116 const next = git(work, ['rev-parse', 'HEAD']).trim();
117 const raw = (type, rev) => ({ type: TYPE[type.toUpperCase()], data: new Uint8Array(gitBuf(work, ['cat-file', type, rev])) });
118 const objects = [raw('commit', 'HEAD'), raw('tree', 'HEAD^{tree}'), ...files.map(f => raw('blob', `HEAD:${f}`))];
119 assert.deepEqual([...new Set(objects.slice(2).map(o => o.data.length))], [8]);
120 const c = GitHttp.create(`${srv.url}/${name}.git`, { tracer: new Tracer() });
121 assert.equal(c.ok, true);
122 const spy = vi.spyOn(Sha1.prototype, 'update');
123 let hashed;
124 try {
125 const r = await c.value.push({ ref: 'refs/heads/main', oldOid: head, newOid: next, objects });
126 hashed = spy.mock.calls.map(([b]) => Buffer.from(b.subarray(0, 32)).toString('latin1')).filter(s => /^(?:blob|tree|commit|tag) \d+\0$/u.test(s));
127 assert.equal(r.ok ? 'ok' : `${r.error.code}: ${r.error.message}`, 'ok');
128 assert.equal(r.value.entries.length, 26); // git's commit, its tree and the 24 blobs: nothing dropped
129 } finally { spy.mockRestore(); }
130 assert.deepEqual(hashed, []);
131 assert.equal(git(bare, ['rev-parse', 'refs/heads/main']).trim(), next);
132 assert.deepEqual(fsckProblems(bare), []);
133});
135// The same count on the second review's own shape (bench-planpush.mjs): blobs of one length filled from a 32-bit LCG,
136// four bytes a step. An LCG's low byte repeats every 256 steps, so bytes read at a regular stride are the same in
137// every blob although no two blobs are equal: a guard that told objects apart by such bytes hashed all of them.
138// Nothing is sent: the fetch throws after the pack is measured and before any chunk of the body is pulled (512 KiB of
139// blobs; a pack over 16 KiB is streamed). README "Errors": such a request is that failure itself, 'network', because
140// the server received no command list ('ambiguous' starts at the first pull). No object is hashed.
141test('push: equal-length blobs that agree at every regular stride are still not hashed', async () => {
142 let seed = 12345;
143 const rnd = () => { seed = (seed * 1664525 + 1013904223) >>> 0; return seed; };
144 const blob = len => { const d = new Uint8Array(len); for (let i = 0; i < len; i += 4) { const r = rnd(); d[i] = r; d[i + 1] = r >>> 8; d[i + 2] = r >>> 16; d[i + 3] = r >>> 24; } return d; };
145 const objects = Array.from({ length: 8 }, () => ({ type: TYPE.BLOB, data: blob(1 << 16) }));
146 // the premise, checked: every blob has the same byte at each multiple of 1024, and no two blobs are equal
147 for (const o of objects) for (let i = 0; i < o.data.length; i += 1024) assert.equal(o.data[i], objects[0].data[0]);
148 assert.equal(new Set(objects.map(o => Buffer.from(o.data).toString('latin1'))).size, 8);
149 const c = GitHttp.create('http://127.0.0.1:9/x.git', { tracer: new Tracer(), fetchImpl: async () => { throw new Error('not sent'); } });
150 assert.equal(c.ok, true);
151 const spy = vi.spyOn(Sha1.prototype, 'update');
152 let hashed;
153 try {
154 const r = await c.value.push({ ref: 'refs/heads/x', oldOid: '0'.repeat(40), newOid: 'a'.repeat(40), objects });
155 hashed = spy.mock.calls.map(([b]) => Buffer.from(b.subarray(0, 32)).toString('latin1')).filter(s => /^(?:blob|tree|commit|tag) \d+\0$/u.test(s));
156 assert.equal(r.ok ? 'ok' : r.error.code, 'network');
157 } finally { spy.mockRestore(); }
158 assert.deepEqual(hashed, []);
159});
161test("the duplicate-object guard is not an export of 'ply/push'", () => {
162 assert.deepEqual(Object.keys(pushEntry).filter(k => /once|dedup|duplicate/iu.test(k)), []);
163});
Diagnostics6 git calls · 161 ms
CallStatusSizeTime0161 ms
discover200191 B18 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=18
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB90 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=83
git.response.bytes=11990
git.response.cache=EXPIRED
git.response.duration_ms=90
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetch2003.9 KiB64 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=64
git.response.bytes=4029
git.response.cache=MISS
git.response.duration_ms=64
git.fetch.wants=365635df914af8bb1249ec11e52f238bd2175c1a
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4029
git.pkt.payload.max=4002
git.sideband.band1.bytes=4002
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e14de08fcea140
Request
8ad0a799…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format