main
1// Cells observed from git: for every cell whose reason git can be made to write, a real `git receive-pack
2// --stateless-rpc` (what git http-backend runs) answers the cell's request in a repository arranged to produce the
3// reason. The request is built here, byte for byte as a client sends it (commands, optional shallow lines, flush, pack
4// from `git pack-objects`), because several reasons need requests ply refuses to send or never sends (shallow lines, a
5// pack that lacks the new commit). For each cell: git's report bytes must equal the cell's derived report
6// (space.gitReport) rendered with the same ref names; ply's outcome on git's bytes, through push() over the fake
7// transport, must be the table's; and the repository's refs must agree with the outcome: landed → every ref moved,
8// rejected → none, partial → exactly the landed ones, ambiguous → recorded (the truth ply did not claim).
9import { spawn, spawnSync } from 'node:child_process';
10import { isMainThread, parentPort, workerData, Worker } from 'node:worker_threads';
11import { availableParallelism } from 'node:os';
12import { mkdirSync, writeFileSync, chmodSync, cpSync, rmSync, unlinkSync } from 'node:fs';
13import { join } from 'node:path';
14import { git, gitBuf, tmpRoot, noMaintenance } from '../server.js';
15import { pkt, FLUSH } from '../../src/pktline.js';
16import { concat, te } from '../../src/bytes.js';
17import * as space from './push-ng.space.js';
19const ENV = { ...Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith('GIT_'))), GIT_AUTHOR_NAME: 'a', GIT_AUTHOR_EMAIL: 'a@x', GIT_COMMITTER_NAME: 'a', GIT_COMMITTER_EMAIL: 'a@x', GIT_AUTHOR_DATE: '1700000000 +0000', GIT_COMMITTER_DATE: '1700000000 +0000', GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' };
20const g = (cwd, args, opts = {}) => git(cwd, args, { env: ENV, ...opts }).trim();
21// permission-based recipes cannot fail as root (CI runners are not root; a root run derives those cells instead)
22const ROOT_USER = process.getuid?.() === 0;
23// the local git's era (space axis `era`): a cell of the other era is derived on this run (CI runs git 2.56, a laptop may run 2.50)
24import { gitVersion } from '../gitversion.js';
25import { unwrap } from '../unwrap.js';
26const [MAJ, MIN] = gitVersion();
27export const LOCAL_ERA = MAJ > 2 || (MAJ === 2 && MIN >= 51) ? 'v2.51+' : 'v2.50';
28export const GIT_VERSION = `${MAJ}.${MIN}`;
29// does this git's receive-pack take --skip-connectivity-check? Asked of git itself (an unknown option exits 129 before
30// it reads a repository), not inferred from the version.
31export const SKIP_CONNECTIVITY = spawnSync('git', ['receive-pack', '--skip-connectivity-check', '--advertise-refs', '/nonexistent-ply-probe'], { env: ENV, encoding: 'utf8' }).status !== 129;
32const BAD_PACK_SEND = 'D+objects';
34// ---- the fixture: one repository of commits, copied per cell ----
35// A ← B ← C, A ← X (diverged), B ← D (never sent: the pack lacks it), B ← W (adds w: in the way of an untracked w);
36// P ← Q ← R: a history a shallow client (boundary Q) pushes.
37function fixture(root) {
38 const src = join(root, 'src');
39 mkdirSync(src); g(src, ['init', '-q', '-b', 'main']);
40 const blob = text => g(src, ['hash-object', '-w', '--stdin'], { input: text });
41 const tree = files => g(src, ['mktree'], { input: Object.entries(files).map(([n, t]) => `100644 blob ${blob(t)}\t${n}\n`).join('') });
42 const commit = (files, msg, parent) => g(src, ['commit-tree', tree(files), ...(parent ? ['-p', parent] : []), '-m', msg]);
43 const A = commit({ f: 'a\n' }, 'A'), B = commit({ f: 'b\n' }, 'B', A), C = commit({ f: 'c\n' }, 'C', B), X = commit({ f: 'x\n' }, 'X', A);
44 const D = commit({ f: 'd\n' }, 'D', B), W = commit({ f: 'b\n', w: 'w\n' }, 'W', B);
45 const P = commit({ p: 'p\n' }, 'P'), Q = commit({ p: 'q\n' }, 'Q', P), R = commit({ p: 'r\n' }, 'R', Q);
46 const oids = { A, B, C, X, D, W, P, Q, R, U: 'dead'.repeat(10), zero: space.ZERO };
47 // objects of a commit that the server (which has A and B) lacks
48 const objectsOf = c => g(src, ['rev-list', '--objects', '--no-object-names', c, `^${B}`, ...(c === R ? [`^${P}`] : [])]).split('\n').filter(Boolean);
49 // D's own objects are sent only by the bad pack recipe that removes D after the connectivity check (BAD_PACK_SEND)
50 const objects = { C: objectsOf(C), X: objectsOf(X), W: objectsOf(W), D: [], R: objectsOf(R), [BAD_PACK_SEND]: objectsOf(D) };
51 // the bare template: refs main, r0, r1, r2 at B, HEAD → main; the non-bare one: the same, main checked out
52 const bare = join(root, 'bare.git');
53 g(root, ['init', '-q', '--bare', '-b', 'main', bare]); noMaintenance(bare);
54 g(src, ['push', '-q', bare, `${B}:refs/heads/main`, `${B}:refs/heads/r0`, `${B}:refs/heads/r1`, `${B}:refs/heads/r2`]);
55 const wt = join(root, 'wt');
56 g(root, ['clone', '-q', bare, wt]); noMaintenance(wt);
57 for (const r of ['r0', 'r1', 'r2']) g(wt, ['update-ref', `refs/heads/${r}`, B]);
58 g(wt, ['remote', 'remove', 'origin']);
59 return { src, oids, objects, bare, wt };
60}
62// ---- recipes: how to make git write each observed reason ----
63// setup(ctx) arranges the copied repository; ctx: { dir (the git dir), work (the worktree or null), names, culprits,
64// cell, oids, hook(name, body), cfg(k, v) }. new: the commit the culprit pushes instead of the shape's. shallow: shallow
65// lines sent before the commands. args: extra receive-pack arguments. pack(bytes): a fault in the pack.
66const sh = body => `#!/bin/sh\n${body}\n`;
67const caseOf = names => names.map(n => n.replace(/[^A-Za-z0-9/_-]/g, '?')).join('|');
68const PROC_HOOK = mode => `#!${process.execPath}
69// a proc-receive hook (receive-pack.c run_proc_receive_hook): version negotiation, then the commands, then a status per command
70const fs = require('node:fs');
71let buf = Buffer.alloc(0);
72const readPkt = () => { while (buf.length < 4) { const b = Buffer.alloc(65536); const n = fs.readSync(0, b); if (!n) return null; buf = Buffer.concat([buf, b.subarray(0, n)]); }
73 const len = parseInt(buf.subarray(0, 4).toString(), 16); if (len === 0) { buf = buf.subarray(4); return ''; }
74 while (buf.length < len) { const b = Buffer.alloc(65536); const n = fs.readSync(0, b); if (!n) return null; buf = Buffer.concat([buf, b.subarray(0, n)]); }
75 const p = buf.subarray(4, len).toString(); buf = buf.subarray(len); return p; };
76const out = s => fs.writeSync(1, s === '' ? '0000' : (s.length + 4).toString(16).padStart(4, '0') + s);
77while (readPkt()) {} out('version=1\\n'); out('');
78const cmds = []; for (let l; (l = readPkt());) cmds.push(l.trim().split(' ')[2]);
79${mode === 'ng' ? "for (const r of cmds) out('ng ' + r + '\\n');" : ''}
80out('');
81`;
82const DENY_UPDATE_INSTEAD = ctx => { ctx.cfg('receive.denyCurrentBranch', 'updateInstead'); };
83const unbornIfCreate = ctx => { if (ctx.cell.command.tag === 'create') { g(ctx.work, ['read-tree', '--empty']); rmSync(join(ctx.work, 'f')); } };
84export const RECIPES = {
85 'eof before pack header was fully read': { pack: b => b.subarray(0, 6) },
86 'protocol error (pack signature mismatch detected)': { pack: b => concat([te.encode('JUNK'), b.subarray(4)]) },
87 'protocol error (pack version unsupported)': { pack: b => { const c = b.slice(); c[7] = 4; return c; } },
88 'unable to create temporary object directory': { root: true, setup: ctx => { chmodSync(join(ctx.dir, 'objects'), 0o555); }, cleanup: ctx => chmodSync(join(ctx.dir, 'objects'), 0o755) },
89 'unpack-objects abnormal exit': { setup: ctx => ctx.cfg('receive.unpackLimit', '1000'), pack: b => { const c = b.slice(); c[c.length - 25] ^= 0xff; return c; } },
90 'index-pack abnormal exit': { setup: ctx => ctx.cfg('receive.unpackLimit', '1'), pack: b => { const c = b.slice(); c[c.length - 25] ^= 0xff; return c; } },
91 'missing necessary objects': { new: 'D' },
92 'deny updating a hidden ref': { setup: ctx => { for (const i of ctx.culprits) ctx.cfg('receive.hideRefs', ctx.names[i], true); } },
93 'deny deleting a hidden ref': { setup: ctx => { for (const i of ctx.culprits) ctx.cfg('receive.hideRefs', ctx.names[i], true); } },
94 'broken symref': { setup: ctx => { for (const i of ctx.culprits) { const n = ctx.names[i]; g(ctx.dir, ['update-ref', '-d', n]); g(ctx.dir, ['symbolic-ref', n, `${n}-loop`]); g(ctx.dir, ['symbolic-ref', `${n}-loop`, n]); } } },
95 'inconsistent aliased update': { setup: ctx => { for (const i of ctx.culprits) { g(ctx.dir, ['update-ref', '-d', ctx.names[i]]); g(ctx.dir, ['symbolic-ref', ctx.names[i], ctx.names[0]]); } } },
96 'shallow update not allowed': { new: 'R', shallow: 'Q' },
97 'funny refname': {},
98 'branch is currently checked out': { work: true, head: true },
99 // update()'s "bad pack": the new object is gone after unpack, which the connectivity check would have refused first.
100 // git with `receive-pack --skip-connectivity-check` (not 2.47.3: "unknown option") skips that check, and the pack
101 // lacks D. Without the option, the same update() check: D is sent, unpacked loose into the quarantine, passes the
102 // connectivity check, and the pre-receive hook (which runs after that check, before the quarantine is migrated)
103 // deletes D's commit object, so update() cannot find it.
104 'bad pack': SKIP_CONNECTIVITY
105 ? { new: 'D', args: ['--skip-connectivity-check'] }
106 : { new: 'D', send: BAD_PACK_SEND, setup: ctx => { ctx.cfg('receive.unpackLimit', '1000'); const d = ctx.oids.D; ctx.hook('pre-receive', sh(`rm "$GIT_QUARANTINE_PATH/${d.slice(0, 2)}/${d.slice(2)}"`)); } },
107 'deletion prohibited': { setup: ctx => ctx.cfg('receive.denyDeletes', 'true') },
108 'deletion of the current branch prohibited': { head: true },
109 'bad ref': { setup: ctx => ctx.cfg('receive.denyNonFastForwards', 'true') },
110 'non-fast-forward': { setup: ctx => ctx.cfg('receive.denyNonFastForwards', 'true') },
111 'hook declined': { setup: ctx => ctx.hook('update', sh(`case "$1" in ${caseOf(ctx.culprits.map(i => ctx.names[i]))}) exit 1;; esac\nexit 0`)) },
112 'Up-to-date check failed': { work: true, head: true, setup: ctx => { DENY_UPDATE_INSTEAD(ctx); unbornIfCreate(ctx); writeFileSync(join(ctx.dir, 'index.lock'), ''); } },
113 'Working directory has unstaged changes': { work: true, head: true, setup: ctx => { DENY_UPDATE_INSTEAD(ctx); writeFileSync(join(ctx.work, 'f'), 'dirty\n'); } },
114 'Working directory has staged changes': { work: true, head: true, setup: ctx => { DENY_UPDATE_INSTEAD(ctx); unbornIfCreate(ctx); writeFileSync(join(ctx.work, 'f'), 'staged\n'); g(ctx.work, ['add', 'f']); } },
115 'Could not update working tree to new HEAD': { work: true, head: true, new: 'W', setup: ctx => { DENY_UPDATE_INSTEAD(ctx); unbornIfCreate(ctx); writeFileSync(join(ctx.work, 'w'), 'untracked\n'); } },
116 'push-to-checkout hook declined': { work: true, head: true, setup: ctx => { DENY_UPDATE_INSTEAD(ctx); ctx.hook('push-to-checkout', sh('exit 1')); } },
117 'failed to update ref#2': {},
118 'atomic transaction failed': {},
119 'pre-receive hook declined': { setup: ctx => ctx.hook('pre-receive', sh('exit 1')) },
120 'unable to migrate objects to permanent storage': { root: true, setup: ctx => { ctx.cfg('receive.unpackLimit', '1'); ctx.hook('pre-receive', sh('chmod a-w objects/pack')); }, cleanup: ctx => chmodSync(join(ctx.dir, 'objects', 'pack'), 0o755) },
121 'fail to run proc-receive hook': { setup: ctx => ctx.cfg('receive.procReceiveRefs', 'refs/for') },
122 'proc-receive failed to report status': { setup: ctx => { ctx.cfg('receive.procReceiveRefs', 'refs/for'); ctx.hook('proc-receive', PROC_HOOK('silent')); } },
123 'failed': { setup: ctx => { ctx.cfg('receive.procReceiveRefs', 'refs/for'); ctx.hook('proc-receive', PROC_HOOK('ng')); } },
124};
125// observable on some git (SPACE.md counts these); observable here: on this run's git, as this user
126export const observableAnywhere = c => !c.reason.reason.derived;
127export const observable = c => observableAnywhere(c) && c.era.tag === LOCAL_ERA && !(RECIPES[c.reason.tag]?.root && ROOT_USER);
129const run = (args, input) => new Promise((resolve, reject) => {
130 const p = spawn('git', args, { env: ENV });
131 const out = [], err = [];
132 p.stdout.on('data', d => out.push(d)); p.stderr.on('data', d => err.push(d));
133 p.stdin.on('error', () => {}); // receive-pack may stop reading a bad pack early
134 p.on('error', reject);
135 p.on('close', code => resolve({ code, out: new Uint8Array(Buffer.concat(out)), err: Buffer.concat(err).toString() }));
136 p.stdin.end(input);
137});
139const refsOf = dir => {
140 const out = new Map();
141 for (const l of g(dir, ['for-each-ref', '--format=%(refname) %(objectname)'], { stdio: ['pipe', 'pipe', 'ignore'] }).split('\n').filter(Boolean)) { const [r, o] = l.split(' '); out.set(r, o); }
142 return out;
143};
145// filter(cell): observe only some cells (debugging: PUSH_NG_ONLY=<reason tag> in push-ng.test.js). shard [k, w]: every w-th.
146async function observeShard({ concurrency = 4, filter = () => true, shard = [0, 1] } = {}) {
147 const { root, cleanup } = tmpRoot();
148 try {
149 const fx = fixture(root);
150 const packCache = new Map();
151 const packFor = news => {
152 const key = [...news].sort().join(',');
153 // a push of D alone sends an empty pack (D is the object it lacks)
154 const list = [...new Set([...news].flatMap(k => fx.objects[k]))];
155 if (!packCache.has(key)) packCache.set(key, news.size ? new Uint8Array(gitBuf(fx.src, ['pack-objects', '--stdout', '-q'], list.map(o => `${o}\n`).join(''))) : null);
156 return packCache.get(key);
157 };
158 const todo = space.cells().filter(c => observable(c) && filter(c)).filter((_, i) => i % shard[1] === shard[0]);
159 const results = [];
160 let next = 0, k = 0;
161 const one = async c => {
162 const rec = RECIPES[c.reason.tag];
163 if (!rec) throw new Error(`no recipe for ${c.reason.tag}`);
164 const cellDir = join(root, `c${k++}`);
165 cpSync(rec.work ? fx.wt : fx.bare, cellDir, { recursive: true });
166 const dir = rec.work ? join(cellDir, '.git') : cellDir, work = rec.work ? cellDir : null;
167 const names = space.refNames(c), culprits = space.culprits(c);
168 const cmds = space.commandsOf(c).map((x, i) => ({ ref: x.ref, old: fx.oids[x.old], new: x.new === 'zero' ? space.ZERO : fx.oids[culprits.includes(i) && rec.new ? rec.new : x.new], newKey: x.new === 'zero' ? null : culprits.includes(i) && rec.new ? rec.new : x.new }));
169 const ctx = {
170 dir, work, names, culprits, cell: c, oids: fx.oids,
171 cfg: (key, v, add = false) => g(dir, ['config', ...(add ? ['--add'] : []), key, v]),
172 hook: (name, body) => { mkdirSync(join(dir, 'hooks'), { recursive: true }); const p = join(dir, 'hooks', name); writeFileSync(p, body); chmodSync(p, 0o755); },
173 };
174 // the shape's server state: a create's ref is absent
175 for (const i of culprits) if ((c.command.tag === 'create' || c.command.tag === 'gone') && /^refs\/heads\/r\d$/.test(names[i])) g(dir, ['update-ref', '-d', names[i]]);
176 if (rec.head) g(dir, ['symbolic-ref', 'HEAD', names[culprits[0]]]);
177 rec.setup?.(ctx);
178 const before = refsOf(dir);
179 const news = new Set(cmds.map(x => (rec.send && x.newKey === rec.new ? rec.send : x.newKey)).filter(Boolean));
180 let pack = packFor(news);
181 if (pack && rec.pack) pack = rec.pack(pack);
182 const caps = ['report-status', ...(c.atomic.tag === 'on' ? ['atomic'] : []), ...(cmds.some(x => x.new === space.ZERO) ? ['delete-refs'] : [])];
183 const req = concat([
184 ...(rec.shallow ? [unwrap(pkt(te.encode(`shallow ${fx.oids[rec.shallow]}\n`)))] : []),
185 ...cmds.map((x, i) => unwrap(pkt(te.encode(`${x.old} ${x.new} ${x.ref}${i === 0 ? '\0' + caps.join(' ') : ''}\n`)))),
186 FLUSH, ...(pack ? [pack] : []),
187 ]);
188 const res = await run(['receive-pack', '--stateless-rpc', ...(rec.args ?? []), dir], req);
189 const after = refsOf(dir);
190 rec.cleanup?.(ctx);
191 if (rec.work) try { unlinkSync(join(dir, 'index.lock')); } catch { /* only Up-to-date check failed leaves one */ }
192 const moved = cmds.filter(x => before.get(x.ref) !== after.get(x.ref)).map(x => x.ref);
193 const commands = cmds.map(x => ({ ref: x.ref, oldOid: x.old, newOid: x.new }));
194 rmSync(cellDir, { recursive: true, force: true });
195 return { id: space.cellId(c), commands, bytes: res.out, stderr: res.err, exit: res.code, moved };
196 };
197 const worker = async () => { while (next < todo.length) { const c = todo[next++]; results.push(await one(c)); } };
198 await Promise.all(Array.from({ length: concurrency }, worker));
199 return results;
200 } finally { cleanup(); }
201}
203// The cells' setup runs git synchronously, so the cells are split across worker threads (PUSH_NG_WORKERS sets how many).
204// filter is a reason tag or null (a function cannot cross into a worker).
205export async function observeAll({ only = null, workers = +(process.env.PUSH_NG_WORKERS ?? Math.min(8, availableParallelism())) } = {}) {
206 const shards = await Promise.all(Array.from({ length: workers }, (_, k) => new Promise((resolve, reject) => {
207 const w = new Worker(new URL(import.meta.url), { workerData: { pushNg: true, k, w: workers, only } });
208 w.once('message', resolve); w.once('error', reject);
209 })));
210 const byId = new Map(space.cells().map(c => [space.cellId(c), c]));
211 const results = shards.flat().map(o => ({ ...o, cell: byId.get(o.id) }));
212 return { results, derived: space.cells().filter(c => !observable(c)), rootUser: ROOT_USER, era: LOCAL_ERA, version: GIT_VERSION };
213}
214if (!isMainThread && workerData?.pushNg) {
215 const { k, w, only } = workerData;
216 observeShard({ shard: [k, w], filter: c => !only || c.reason.tag === only }).then(r => parentPort.postMessage(r));
217}
Diagnostics4 git calls · 266 ms
CallStatusSizeTime0266 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2006.6 KiB260 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=260 git.response.bytes=6725 git.response.cache=MISS git.response.duration_ms=260 git.fetch.wants=b9ca8d8866c3595f1dea9de26ed1b5a06d4a07fb git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=6725 git.pkt.payload.max=6704 git.sideband.band1.bytes=6703 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e1e2da393cd96e
- Request
- bba767c5…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format