GitHubivarvong
ply
main
1// A push whose pack is past INFLATE_CHUNK goes out as a stream (#172): pass 1 measures, pass 2 is the request body.
2// Against `git http-backend` (the oracle: the ref moves, fsck is clean, the objects are the ones sent):
3// - the body is a ReadableStream with duplex 'half', chunked, of the length span git.request.bytes says
4// - a pack at most INFLATE_CHUNK is sent as bytes with a Content-Length, as before
5// - over maxPackBytes: limit_exceeded with no receive-pack request, `got` the measured pack
6// - an object that changes between the passes: pass 2 refuses to finish the pack; the push is ambiguous (original
7// internal) and git did not move the ref (it never got a whole pack)
8// - a 3xx on the streamed POST is not followed and nothing reaches the other origin
9// - pushRepo's delta pack streams the same way
10// - memory: a 32 MiB push holds well under one copy of the object (child process, --expose-gc)
11import { test, beforeAll as before, afterAll as after } from 'vitest';
12import assert from 'node:assert/strict';
13import http from 'node:http';
14import { spawn } from 'node:child_process';
15import { MEMORY_CELL_FLAGS } from './memcell.js';
16import { randomFillSync } from 'node:crypto';
17import { join } from 'node:path';
18import { mkdirSync, writeFileSync } from 'node:fs';
19import { git, tmpRoot, makeBareRepo, startGitServer, fsckProblems, initRepo } from './server.js';
20import { createTransport } from '../src/http.js';
21import { push, ZERO_OID } from '../src/push.js';
22import { pushRepo } from '../src/repo-node.js';
23import { hashObject, encodeTree, encodeCommit, TYPE } from '../src/objects.js';
24import { withLimits } from '../src/limits.js';
25import { INFLATE_CHUNK } from '../src/format.js';
26import { Tracer, arrayExporter, flattenLine } from '../src/otel.js';
27import { unwrap } from './unwrap.js';
28import { readBody } from './body.js';
30let root, cleanup, srv, n = 0;
31before(async () => { ({ root, cleanup } = tmpRoot()); srv = await startGitServer(root); });
32after(async () => { await srv.close(); cleanup(); });
34const random = size => { const d = new Uint8Array(size); for (let i = 0; i < size; i += 65536) randomFillSync(d, i, Math.min(65536, size - i)); return d; };
35const commitOf = blob => {
36 const tree = unwrap(encodeTree([{ mode: '100644', name: 'big.bin', oid: hashObject('blob', blob) }]));
37 const commit = unwrap(encodeCommit({ tree: hashObject('tree', tree), author: 'a <a@x> 1700000000 +0000', message: 'm\n' }));
38 return { oid: hashObject('commit', commit), objects: [{ type: TYPE.BLOB, data: blob }, { type: TYPE.TREE, data: tree }, { type: TYPE.COMMIT, data: commit }] };
39};
40const repo = () => { const name = `s${n++}`; return { bare: makeBareRepo(root, name), url: `${srv.url}/${name}.git` }; };
41const tipOf = (bare, ref = 'refs/heads/main') => { try { return git(bare, ['rev-parse', '--verify', '-q', ref]).trim(); } catch { return null; } };
42// every receive-pack request: whether its body was a stream, its duplex, its bytes; then the real request
43const capture = (before = () => {}) => {
44 const seen = [];
45 const fetchImpl = async (url, init) => {
46 if (!String(url).endsWith('/git-receive-pack')) return fetch(url, init);
47 before(init);
48 const streamed = typeof init.body?.getReader === 'function';
49 seen.push({ streamed, duplex: init.duplex ?? null, redirect: init.redirect });
50 return fetch(url, init);
51 };
52 return { seen, fetchImpl };
53};
54const traced = () => { const ex = arrayExporter(); return { tracer: new Tracer({ exporter: ex }), spans: () => ex.lines.flatMap(flattenLine) }; };
56test('a pack past INFLATE_CHUNK is streamed: duplex half, redirect error, chunked; git takes it', async () => {
57 const { bare, url } = repo();
58 const c = commitOf(random(4 << 20));
59 const { seen, fetchImpl } = capture();
60 const { tracer, spans } = traced();
61 const t = unwrap(createTransport(url, { fetchImpl, tracer }));
62 const r = await push(t, { ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: c.oid, objects: c.objects });
63 assert.equal(r.ok, true, r.error?.message);
64 assert.deepEqual(seen, [{ streamed: true, duplex: 'half', redirect: 'error' }]);
65 assert.equal(tipOf(bare), c.oid);
66 assert.deepEqual(fsckProblems(bare), []);
67 const sent = spans().find(s => s.name === 'git.receive-pack');
68 const exchange = sent;
69 assert.equal(sent.attrs['git.push.pack.bytes'], r.value.packBytes);
70 // the commands' pkt-line (oid oid ref\0caps\n) and the flush, then the pack
71 assert.ok(exchange.attrs['git.request.bytes'] > r.value.packBytes && exchange.attrs['git.request.bytes'] < r.value.packBytes + 400, `${exchange.attrs['git.request.bytes']}`);
72});
74test('the server reads the streamed body chunked, and the bytes it got are the length declared', async () => {
75 const { bare } = repo();
76 const name = bare.split('/').pop();
77 let got = null;
78 const front = http.createServer((req, res) => {
79 if (!req.url.endsWith('/git-receive-pack')) { req.pipe(http.request(`${srv.url}${req.url}`, { method: req.method, headers: req.headers }, up => { res.writeHead(up.statusCode, up.headers); up.pipe(res); })); return; }
80 const chunks = [];
81 req.on('data', d => chunks.push(d));
82 req.on('end', () => {
83 got = { te: req.headers['transfer-encoding'] ?? null, cl: req.headers['content-length'] ?? null, bytes: Buffer.concat(chunks) };
84 const { 'transfer-encoding': _te, ...headers } = req.headers;
85 const up = http.request(`${srv.url}${req.url}`, { method: 'POST', headers: { ...headers, 'content-length': String(got.bytes.length) } }, u => { res.writeHead(u.statusCode, u.headers); u.pipe(res); });
86 up.end(got.bytes);
87 });
88 });
89 await new Promise(r => front.listen(0, '127.0.0.1', r));
90 try {
91 const c = commitOf(random(INFLATE_CHUNK * 4));
92 const { tracer, spans } = traced();
93 const t = unwrap(createTransport(`http://127.0.0.1:${front.address().port}/${name}`, { tracer }));
94 const r = await push(t, { ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: c.oid, objects: c.objects });
95 assert.equal(r.ok, true, r.error?.message);
96 assert.equal(got.te, 'chunked');
97 assert.equal(got.cl, null);
98 const exchange = spans().find(s => s.name === 'git.receive-pack');
99 assert.equal(got.bytes.length, exchange.attrs['git.request.bytes']);
100 assert.equal(tipOf(bare), c.oid);
101 } finally { await new Promise(r => front.close(r)); }
102});
104test('a pack at most INFLATE_CHUNK is sent as bytes, redirect manual, as before', async () => {
105 const { bare, url } = repo();
106 const c = commitOf(new TextEncoder().encode('small\n'));
107 const { seen, fetchImpl } = capture();
108 const t = unwrap(createTransport(url, { fetchImpl }));
109 const r = await push(t, { ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: c.oid, objects: c.objects });
110 assert.equal(r.ok, true, r.error?.message);
111 assert.deepEqual(seen, [{ streamed: false, duplex: null, redirect: 'manual' }]);
112 assert.ok(r.value.packBytes <= INFLATE_CHUNK);
113 assert.equal(tipOf(bare), c.oid);
114});
116test('a streamed pack over maxPackBytes is refused with no receive-pack request; got is the measured pack', async () => {
117 const { bare, url } = repo();
118 const c = commitOf(random(1 << 20));
119 const { seen, fetchImpl } = capture();
120 const t = unwrap(createTransport(url, { fetchImpl, limits: withLimits({ maxPackBytes: 1 << 20 }) }));
121 const r = await push(t, { ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: c.oid, objects: c.objects });
122 assert.equal(r.ok, false);
123 assert.deepEqual([r.error.code, r.error.what, r.error.max], ['limit_exceeded', 'push pack bytes', 1 << 20]);
124 assert.ok(r.error.got > 1 << 20);
125 assert.equal(seen.length, 0);
126 assert.equal(tipOf(bare), null);
127});
129test('an object that changes between the passes: pass 2 refuses, the push is ambiguous, git did not move the ref', async () => {
130 for (const shape of ['same length', 'longer']) {
131 const { bare, url } = repo();
132 // 'longer': random then zeros (pass 1 is past INFLATE_CHUNK, so it streams), the zeros random by pass 2
133 const blob = random(INFLATE_CHUNK * 8);
134 if (shape === 'longer') blob.fill(0, INFLATE_CHUNK * 2);
135 const c = commitOf(blob);
136 // after pass 1 (the request is being made), before pass 2 reads the blob
137 const { seen, fetchImpl } = capture(() => { if (shape === 'longer') randomFillSync(blob, INFLATE_CHUNK * 2); else blob[100] ^= 1; });
138 const t = unwrap(createTransport(url, { fetchImpl }));
139 const r = await push(t, { ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: c.oid, objects: c.objects });
140 assert.equal(r.ok, false, shape);
141 assert.equal(r.error.code, 'ambiguous', shape);
142 assert.equal(r.error.original, 'internal', shape);
143 assert.match(r.error.message, /second deflate pass/, shape);
144 assert.equal(seen.length, 1, shape);
145 assert.equal(tipOf(bare), null, `${shape}: git refused the cut pack`);
146 }
147});
149test('a 3xx on the streamed POST is not followed: nothing reaches the other origin, the push is ambiguous', async () => {
150 let other = 0;
151 const target = http.createServer((req, res) => { other++; req.resume(); res.end(); });
152 await new Promise(r => target.listen(0, '127.0.0.1', r));
153 const { bare } = repo();
154 const name = bare.split('/').pop();
155 const front = http.createServer((req, res) => {
156 if (req.method === 'POST') { req.resume(); res.writeHead(307, { location: `http://127.0.0.1:${target.address().port}/x/git-receive-pack` }); res.end(); return; }
157 req.pipe(http.request(`${srv.url}${req.url}`, { method: req.method, headers: req.headers }, up => { res.writeHead(up.statusCode, up.headers); up.pipe(res); }));
158 });
159 await new Promise(r => front.listen(0, '127.0.0.1', r));
160 try {
161 const c = commitOf(random(INFLATE_CHUNK * 3));
162 const t = unwrap(createTransport(`http://127.0.0.1:${front.address().port}/${name}`));
163 const r = await push(t, { ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: c.oid, objects: c.objects });
164 assert.equal(r.ok, false);
165 assert.equal(r.error.code, 'ambiguous');
166 assert.equal(r.error.original, 'network');
167 assert.equal(other, 0);
168 assert.equal(tipOf(bare), null);
169 } finally { await new Promise(r => front.close(r)); await new Promise(r => target.close(r)); }
170});
172test('pushRepo streams its delta pack: the history lands, fsck clean', async () => {
173 const { bare, url } = repo();
174 const work = join(root, `w${n++}`); mkdirSync(work);
175 initRepo(work);
176 const base = Buffer.from(random(512 << 10));
177 for (let i = 0; i < 3; i++) {
178 base.write(`version ${i}\n`, 1000 * i);
179 writeFileSync(join(work, 'data.bin'), base);
180 git(work, ['add', '.']); git(work, ['-c', 'user.name=L', '-c', 'user.email=l@x', 'commit', '-q', '-m', `c${i}`]);
181 }
182 const { seen, fetchImpl } = capture();
183 const r = await pushRepo(work, url, { refspecs: ['main'] }, { fetchImpl });
184 assert.equal(r.ok, true, r.error?.message);
185 assert.equal(r.value.deltas, 2, 'two OFS deltas against the previous version');
186 assert.deepEqual(seen.map(s => s.streamed), [true]);
187 assert.equal(tipOf(bare), git(work, ['rev-parse', 'main']).trim());
188 assert.deepEqual(fsckProblems(bare), []);
189});
191test('the bytes of a captured streamed body are the request a buffered push would send', async () => {
192 const { url } = repo();
193 const c = commitOf(random(INFLATE_CHUNK * 2));
194 let bytes = null;
195 const fetchImpl = async (u, init) => { if (!String(u).endsWith('/git-receive-pack')) return fetch(u, init); const b = await readBody(init); bytes = b.bytes; assert.equal(b.streamed, true); return fetch(u, b.init); };
196 const t = unwrap(createTransport(url, { fetchImpl }));
197 assert.equal((await push(t, { ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: c.oid, objects: c.objects })).ok, true);
198 const { buildPack } = await import('../src/pack.js');
199 const pack = (await buildPack(c.objects)).pack;
200 assert.deepEqual(bytes.subarray(bytes.length - pack.length), Buffer.from(pack));
201 assert.equal(bytes.subarray(bytes.length - pack.length - 4, bytes.length - pack.length).toString(), '0000');
202});
204const runChild = (args) => new Promise((resolve, reject) => {
205 const child = spawn(process.execPath, [...MEMORY_CELL_FLAGS, '--import', './test/ts-resolve.mjs', 'test/push-stream-child.mjs', ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
206 let out = '', errText = '';
207 child.stdout.on('data', d => { out += d; });
208 child.stderr.on('data', d => { errText += d; });
209 child.on('close', status => { try { resolve({ status, ...JSON.parse(out) }); } catch { resolve({ status, out, err: errText }); } });
210 child.on('error', reject);
211});
213test('memory: a 32 MiB streamed push holds well under one copy of the object (live arrayBuffers, gc-sampled)', async () => {
214 const { bare, url } = repo();
215 const L = 32 << 20;
216 const r = await runChild([url, String(L)]);
217 assert.equal(r.status, 0, r.err ?? r.out);
218 console.log(JSON.stringify(r));
219 assert.equal(r.ok, true, r.error);
220 assert.equal(tipOf(bare), r.commit);
221 // measured: 80 KB to 2.3 MB live beyond the object (socket and zlib buffers in flight); the buffered writer held about 4 L
222 assert.ok(r.liveDelta < L / 8, `live arrayBuffers delta ${r.liveDelta}`);
223 assert.ok(r.rssDelta < L, `rss delta ${r.rssDelta}`);
224});
Diagnostics6 git calls · 161 ms
CallStatusSizeTime0161 ms
discover200191 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=11990 git.response.cache=UPDATING git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=150 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetch2004.0 KiB150 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=4122 git.response.cache=MISS git.response.duration_ms=150 git.fetch.wants=45ee229d4b4ebbf28c47d1a83141161f521b609c git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4122 git.pkt.payload.max=4095 git.sideband.band1.bytes=4095 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e0fb02ec26332c
- Request
- edf00ab3…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format