Artifactspublic-v0

ply

main
ply4 git calls407 ms48 KiB2 cachedprotocol v2
614 lines40 KiBRaw
1// Resumable transfer (#238): one commit's tree, from a spooled pack, to a fresh repository as a new root commit, in
2// pushes under a cap. The oracle is git http-backend with receive.fsckObjects and transfer.fsckObjects on:
3// - end to end (REF-only and OFS packs, several batches): the ref is the planned head, no temp ref is left,
4// fsck --strict is clean, exactly the tree's objects and the one new commit are reachable, ls-tree -r is the
5// source's, the commit has no parents and keeps the source's author, committer and message
6// - an OFS pack whose delta chain a small budget cuts below a delta base (crossBatchRef): done in one run and after
7// a pause and resume, no temp ref, the same checks
8// - resume: paused after every single push and resumed, the same head; a push that dies mid-stream (the spool
9// fails) leaves the refs where they were and the next run finishes; a step already landed, sent again, is refused
10// and moves nothing
11// - refusals before any push: a budget an object cannot fit, a tree not in the pack, a commit that does not parse;
12// at a step: an edited plan, another pack, other resolved ids, other commit bytes, a step not in the plan; at
13// status: the ref already at another commit, a temp ref the plan never writes; a step that never lands
14// - the commit policy (signatures dropped, other headers kept) and the deterministic ids
15// - sweepTransfers removes what an abandoned transfer left, keeping what it is told to; any prefix but ply's own
16// temp-ref namespace is refused before a request, and only names of a temp ref's shape are deleted (#298)
17// - the plan's types: batch steps carry their temp commit, the final step none; prev is an Option; refs are refused
18// unless git would push them; a plan read back with a malformed step, id or ref is refused
19// - ls-refs that lags a push (a host that acknowledges before its reads show it): settled by reading the refs again,
20// never by sending the step again; a lag longer than the reads pauses the run ('settling'); a step sent again after a
21// lagging read is refused by the server, and the refs, settled, show it landed, so the run goes on
22// - a done run reports a temp ref left behind (tempLeft); the spans each function writes, with the plan hash
23import { test, beforeAll as before, afterAll as after } from 'vitest';
24import assert from 'node:assert/strict';
25import { mkdirSync, writeFileSync } from 'node:fs';
26import { join } from 'node:path';
27import { randomFillSync } from 'node:crypto';
28import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer, fsckProblems, initRepo } from './server.js';
29import { scanPackIndex } from '../src/packscanindex.js';
30import { resolvePackRange } from '../src/packrangeresolve.js';
31import { rangeFromBytes } from '../src/rangesource.js';
32import { createTransport } from '../src/http.js';
33import { planTransfer, checkTransferPlan, rootCommit, batchTrees, TRANSFER_REF_PREFIX } from '../src/transferplan.js';
34import { transferStatus, transferStep, runTransfer, copyTree, sweepTransfers } from '../src/transfer.js';
35import { te } from '../src/bytes.js';
36import { TYPE } from '../src/objects.js';
37import { Tracer, arrayExporter, flattenLine } from '../src/otel.js';
38import { unwrap } from './unwrap.js';
40const must = r => { if (r.ok) return r.value; throw Object.assign(new Error(r.error.message), { code: r.error.code }); };
41const streamOf = bytes => new ReadableStream({ start(c) { c.enqueue(bytes); c.close(); } });
42const ident = ['-c', 'user.name=Ann Author', '-c', 'user.email=ann@x'];
43const REF = 'refs/heads/main';
44// 300 objects a push (258 of them ply's generated objects), 200 KB a request: several batches of ~40 KB blobs
45const BUDGET = { maxRequestBytes: 200_000, maxObjects: 300, maxObjectBytes: 1 << 20, maxCompressedObjectBytes: 1 << 20, reserveBytes: 0, reserveObjects: 0 };
47let root, cleanup, srv, n = 0, work, src;
48before(async () => {
49 ({ root, cleanup } = tmpRoot());
50 srv = await startGitServer(root);
51 work = join(root, 'source');
52 mkdirSync(join(work, 'a', 'b'), { recursive: true });
53 initRepo(work);
54 for (let i = 0; i < 10; i++) {
55 const blob = new Uint8Array(40_000 + i * 500);
56 randomFillSync(blob);
57 writeFileSync(join(work, i % 3 === 0 ? 'a' : i % 3 === 1 ? join('a', 'b') : '.', `r${i}.bin`), blob);
58 }
59 // near-copies, so the pack holds delta chains (REF on REF in the REF-only pack)
60 const base = new Uint8Array(30_000);
61 randomFillSync(base);
62 for (let v = 0; v < 5; v++) { base[v * 991] ^= 0xff; writeFileSync(join(work, 'a', `chain${v}.bin`), base); }
63 writeFileSync(join(work, 'small.txt'), 'hello\n');
64 git(work, ['add', '.']);
65 git(work, [...ident, 'commit', '-q', '-m', 'one']);
66 writeFileSync(join(work, 'small.txt'), 'hello again\n');
67 git(work, ['add', '.']);
68 git(work, [...ident, 'commit', '-q', '-m', 'two\n\nbody line\n']);
69 const head = git(work, ['rev-parse', 'HEAD']).trim();
70 // a depth-1 pack: the commit and its tree's objects, nothing of the parent
71 const objs = git(work, ['rev-list', '--objects', '--no-walk', head]);
72 src = {
73 head,
74 commit: new Uint8Array(gitBuf(work, ['cat-file', 'commit', head])),
75 lsTree: git(work, ['ls-tree', '-r', head]),
76 tree: git(work, ['rev-parse', `${head}^{tree}`]).trim(),
77 packs: {
78 ref: new Uint8Array(gitBuf(work, ['pack-objects', '--stdout', '-q', '--depth=50'], objs)),
79 ofs: new Uint8Array(gitBuf(work, ['pack-objects', '--stdout', '-q', '--depth=50', '--delta-base-offset'], objs)),
80 },
81 };
82});
83after(async () => { await srv?.close(); cleanup?.(); });
85async function spooled(pack) {
86 const p = must(await scanPackIndex(streamOf(pack)));
87 const source = rangeFromBytes(pack);
88 return { source, pack: p, resolved: must(await resolvePackRange(p, source)) };
90function repo(options = {}) {
91 const name = `t${n++}`;
92 const bare = makeBareRepo(root, name);
93 return { bare, t: unwrap(createTransport(`${srv.url}/${name}.git`, options)) };
95// A destination whose ls-refs lags its pushes: after each receive-pack, the next `lag()` ls-refs answer what the read
96// before the push saw (the refs as they were). Everything else is the real transport.
97function lagging(t, lag) {
98 let stale = 0, last = null, reads = 0;
99 const view = Object.create(t);
100 view.frames = function frames(service, body, span, hints) {
101 const real = t.frames.call(this, service, body, span, hints);
102 if (service !== 'git-upload-pack') { stale = lag(); return real; }
103 reads++;
104 if (stale > 0 && last !== null) { stale--; const kept = last; return (async function* replay() { yield* kept; })(); }
105 return (async function* record() { const got = []; for await (const f of real) { got.push(f); yield f; } last = got; })();
106 };
107 return { t: view, reads: () => reads };
109const refsOf = bare => git(bare, ['for-each-ref', '--format=%(refname) %(objectname)']).trim();
110const objectsOf = (dir, rev) => git(dir, ['rev-list', '--objects', rev]).split('\n').filter(Boolean).map(l => l.slice(0, 40)).sort();
112function assertCopied(bare, plan) {
113 assert.equal(git(bare, ['rev-parse', REF]).trim(), plan.head);
114 assert.equal(git(bare, ['for-each-ref', 'refs/ply']).trim(), '', 'no temp ref left');
115 assert.deepEqual(fsckProblems(bare, { args: ['--strict'] }), []);
116 assert.equal(git(bare, ['ls-tree', '-r', REF]), src.lsTree);
117 assert.equal(git(bare, ['rev-list', '--parents', '-n1', REF]).trim(), plan.head, 'a root commit');
118 const want = [...git(work, ['rev-list', '--objects', '--no-walk', src.tree]).split('\n').filter(Boolean).map(l => l.slice(0, 40)), plan.head].sort();
119 assert.deepEqual(objectsOf(bare, REF), want, 'exactly the tree and the new commit');
120 for (const f of ['%an <%ae> %at %ai', '%cn <%ce> %ct', '%B']) assert.equal(git(bare, ['log', '-1', `--format=${f}`, REF]), git(work, ['log', '-1', `--format=${f}`, src.head]));
123test('end to end, REF-only and OFS packs: several pushes under the cap, fsck --strict, exactly the tree, a root commit', async () => {
124 for (const kind of ['ref', 'ofs']) {
125 const from = await spooled(src.packs[kind]);
126 const { bare, t } = repo();
127 const { plan, run } = must(await copyTree(t, from, { budget: BUDGET, ref: REF, commit: src.commit }));
128 assert.ok(plan.steps.length >= 4, `${kind}: ${plan.steps.length} steps`);
129 assert.equal(run.tag, 'done');
130 assert.equal(run.pushes, plan.steps.length);
131 assert.equal(plan.tempRef.startsWith(TRANSFER_REF_PREFIX), true);
132 for (const b of plan.pack.batches) assert.ok(b.packBytes <= BUDGET.maxRequestBytes);
133 assertCopied(bare, plan);
134 // deterministic: planned again, the same plan and hash
135 assert.deepEqual(must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit })), plan);
136 // a plan survives JSON (a store between Worker requests)
137 assert.ok(checkTransferPlan(JSON.parse(JSON.stringify(plan)), { ref: REF }).ok);
138 assert.deepEqual(must(await transferStatus(t, plan, { ref: REF })), { tag: 'done', head: plan.head, tempLeft: false });
139 }
140});
142test('an OFS pack cut below a delta base (crossBatchRef): the transfer finishes and resumes; git holds exactly the tree', async () => {
143 const from = await spooled(src.packs.ofs);
144 const c = from.pack.columns;
145 // 258 generated objects and 3 of the pack's entries a push (the three trees need that): the chain0..4 family (a blob
146 // and deltas on deltas) cannot fit one push, so it is refused without crossBatchRef and cut in chain order with it
147 const budget = { ...BUDGET, maxObjects: 261 };
148 assert.equal(planTransfer(from.pack, from.resolved, { budget, ref: REF, commit: src.commit }).error.code, 'limit_exceeded');
149 const plan = must(planTransfer(from.pack, from.resolved, { budget, ref: REF, commit: src.commit, crossBatchRef: true }));
150 // the fixture, read from git's pack and the plan's batches: an OFS_DELTA whose base is itself an OFS_DELTA, pushed in
151 // an earlier batch. Its base id is not in the scan index (a delta's id is known only once it is applied).
152 const batchOf = new Map(plan.pack.batches.flatMap((b, k) => b.entries.map(i => [i, k])));
153 const cut = [...batchOf.keys()].filter(i => c.storedType[i] === TYPE.OFS_DELTA && c.storedType[c.baseIndex[i]] === TYPE.OFS_DELTA && batchOf.get(c.baseIndex[i]) < batchOf.get(i));
154 assert.ok(cut.length >= 1, `OFS deltas cut from a delta base: ${cut.length}`);
155 const lastCut = Math.max(...cut.map(i => batchOf.get(i)));
156 // one run to the end
157 const whole = repo();
158 const run = must(await runTransfer(whole.t, from, plan, src.commit, { ref: REF }));
159 assert.deepEqual(run, { tag: 'done', head: plan.head, pushes: plan.steps.length, tempLeft: false });
160 assertCopied(whole.bare, plan);
161 // paused before the last cut batch, then resumed: the temp ref is there in between, gone at the end
162 const { bare, t } = repo();
163 const paused = must(await runTransfer(t, from, plan, src.commit, { maxSteps: lastCut, ref: REF }));
164 assert.equal(paused.tag, 'paused');
165 assert.equal(paused.pushes, lastCut);
166 assert.match(refsOf(bare), new RegExp(`^${plan.tempRef} [0-9a-f]{40}$`, 'u'));
167 const resumed = must(await runTransfer(t, from, plan, src.commit, { ref: REF }));
168 assert.equal(resumed.tag, 'done');
169 assert.equal(resumed.pushes, plan.steps.length - lastCut);
170 assert.equal(resumed.tempLeft, false);
171 assertCopied(bare, plan);
172});
174test('a `resolved` whose oid() answers an id, then something else: one id asked per rewritten entry; too few is a coded refusal, never a rejection', async () => {
175 const from = await spooled(src.packs.ofs);
176 const c = from.pack.columns;
177 const budget = { ...BUDGET, maxObjects: 261 };
178 const plan = must(planTransfer(from.pack, from.resolved, { budget, ref: REF, commit: src.commit, crossBatchRef: true }));
179 // from git's pack columns and the plan: an OFS_DELTA whose base is not in its own batch is rewritten as a REF_DELTA,
180 // and each rewrite needs one id. Counted per batch.
181 const batchOf = new Map(plan.pack.batches.flatMap((b, k) => b.entries.map(i => [i, k])));
182 const rewrites = plan.pack.batches.map((b, k) => b.entries.filter(i => c.storedType[i] === TYPE.OFS_DELTA && batchOf.get(c.baseIndex[i]) !== k).length);
183 const total = rewrites.reduce((a, b) => a + b, 0);
184 assert.ok(total >= 1, `rewrites: ${total}`);
185 const lastCut = rewrites.findLastIndex(r => r > 0);
186 const isRewrite = (i, k) => c.storedType[i] === TYPE.OFS_DELTA && batchOf.get(c.baseIndex[i]) !== k;
187 // How often each entry's id is needed to push `steps`, by hand: once for its line in its batch's temp tree
188 // (transferplan's entryAt, as on main), and once for each REF_DELTA header that names it as a base.
189 const needed = steps => {
190 const m = new Map();
191 const add = i => m.set(i, (m.get(i) ?? 0) + 1);
192 for (const k of steps) for (const i of plan.pack.batches[k].entries) { add(i); if (isRewrite(i, k)) add(c.baseIndex[i]); }
193 return m;
194 };
195 // an index that answers honestly as often as `allow` says (one time fewer for `shortOn`), then with what is not an id
196 const flipping = (allow, shortOn = -1) => {
197 const state = { lies: 0 };
198 const asked = new Map();
199 const oid = i => {
200 const nth = (asked.get(i) ?? 0) + 1;
201 asked.set(i, nth);
202 if (nth <= (allow.get(i) ?? 0) - (i === shortOn ? 1 : 0)) return from.resolved.oid(i);
203 state.lies++;
204 return { tag: 'some', value: 'z'.repeat(40) };
205 };
206 return { state, from: { ...from, resolved: { ...from.resolved, oid } } };
207 };
208 const everyStep = plan.pack.batches.map((_, k) => k);
209 // a base a rewritten entry of the last cut batch rests on
210 const lastBase = c.baseIndex[plan.pack.batches[lastCut].entries.find(i => isRewrite(i, lastCut))];
211 // runTransfer with exactly the answers needed: done without one lie told, and git holds exactly the tree
212 const whole = repo();
213 const enough = flipping(needed(everyStep));
214 const run = await runTransfer(whole.t, enough.from, plan, src.commit, { ref: REF });
215 assert.equal(run.ok, true, run.ok ? '' : `${run.error.code}: ${run.error.message}`);
216 assert.deepEqual(run.value, { tag: 'done', head: plan.head, pushes: plan.steps.length, tempLeft: false });
217 assert.equal(enough.state.lies, 0, 'an id was asked for more often than once per use');
218 assertCopied(whole.bare, plan);
219 // one honest answer too few for that base: the run ends with the builder's refusal (a Result); the ref is not created
220 const short = repo();
221 const few = await runTransfer(short.t, flipping(needed(everyStep), lastBase).from, plan, src.commit, { ref: REF });
222 assert.equal(few.ok, false);
223 assert.equal(few.error.code, 'bad_request');
224 assert.equal(few.error.what, 'pack subset base');
225 assert.equal(git(short.bare, ['for-each-ref', REF]).trim(), '');
226 // transferStep on the last cut batch: too few answers is the same refusal and nothing moves; enough, and it lands
227 const { bare, t } = repo();
228 must(await runTransfer(t, from, plan, src.commit, { maxSteps: lastCut, ref: REF }));
229 const step = must(await transferStatus(t, plan, { ref: REF })).step;
230 assert.equal(step, lastCut);
231 const before = refsOf(bare);
232 const refused = await transferStep(t, flipping(needed([lastCut]), lastBase).from, plan, step, { commit: src.commit, ref: REF });
233 assert.equal(refused.ok, false);
234 assert.equal(refused.error.code, 'bad_request');
235 assert.equal(refused.error.what, 'pack subset base');
236 assert.equal(refsOf(bare), before);
237 const landed = await transferStep(t, flipping(needed([lastCut])).from, plan, step, { commit: src.commit, ref: REF });
238 assert.equal(landed.ok, true, landed.ok ? '' : `${landed.error.code}: ${landed.error.message}`);
239 assert.equal(must(await runTransfer(t, from, plan, src.commit, { ref: REF })).tag, 'done');
240 assertCopied(bare, plan);
241});
243test('everything in one push: no temp ref; atomic off: one plain push', async () => {
244 const from = await spooled(src.packs.ref);
245 const big = { ...BUDGET, maxRequestBytes: 10_000_000 };
246 for (const atomic of [true, false]) {
247 const { bare, t } = repo();
248 const { plan, run } = must(await copyTree(t, from, { budget: big, ref: REF, commit: src.commit, atomic }));
249 assert.equal(plan.steps.length, 1);
250 assert.equal(run.pushes, 1);
251 assertCopied(bare, plan);
252 }
253 const { bare, t } = repo();
254 const { plan } = must(await copyTree(t, from, { budget: BUDGET, ref: REF, commit: src.commit, atomic: false }));
255 assertCopied(bare, plan);
256});
258test('resume: paused after every push and resumed, the same head; a step already landed, sent again, moves nothing', async () => {
259 const from = await spooled(src.packs.ref);
260 const plan = must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit }));
261 const { bare, t } = repo();
262 let runs = 0;
263 for (;;) {
264 const r = must(await runTransfer(t, from, plan, src.commit, { maxSteps: 1, ref: REF }));
265 runs++;
266 if (r.tag === 'done') break;
267 assert.equal(r.pushes, 1);
268 const at = r.step;
269 // the step the destination is on, sent again: refused (stale ref), the refs unchanged
270 if (at > 0) {
271 const before = refsOf(bare);
272 const again = await transferStep(t, from, plan, at - 1, { commit: src.commit, ref: REF });
273 assert.equal(again.ok, false);
274 assert.equal(refsOf(bare), before);
275 }
276 }
277 // one push a run; the run that pushes the last step reads the refs back and is done
278 assert.equal(runs, plan.steps.length);
279 assertCopied(bare, plan);
280});
282test('a push that dies mid-stream moves nothing; the next run finishes; a step that never lands is ambiguous', async () => {
283 const from = await spooled(src.packs.ref);
284 const plan = must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit }));
285 const { bare, t } = repo();
286 must(await runTransfer(t, from, plan, src.commit, { maxSteps: 2, ref: REF }));
287 const before = refsOf(bare);
288 // a spool that fails partway through every read: the pack stream errors, the push cannot have landed
289 const dying = { ...from, source: { size: from.source.size, read: async (o, l) => {
290 const r = await from.source.read(o, l);
291 if (!r.ok) return r;
292 let sent = 0;
293 const reader = r.value.getReader();
294 return { ok: true, value: new ReadableStream({ async pull(c) {
295 const { done, value } = await reader.read();
296 if (done) { c.close(); return; }
297 sent += value.length;
298 if (sent > l / 2 && l > 1000) { c.error(new Error('spool died')); return; }
299 c.enqueue(value);
300 } }) };
301 } } };
302 const died = await transferStep(t, dying, plan, must(await transferStatus(t, plan, { ref: REF })).step, { commit: src.commit, ref: REF });
303 assert.equal(died.ok, false);
304 assert.equal(refsOf(bare), before, 'the refs where they were');
305 const stuck = await runTransfer(t, dying, plan, src.commit, { retries: 1, ref: REF });
306 assert.equal(stuck.ok, false);
307 assert.equal(stuck.error.code, 'ambiguous');
308 assert.match(stuck.error.message, /did not land after 2 tries/);
309 assert.equal(refsOf(bare), before);
310 const done = must(await runTransfer(t, from, plan, src.commit, { ref: REF }));
311 assert.equal(done.tag, 'done');
312 assertCopied(bare, plan);
313});
315test('refusals before any push: an object over the budget, a tree not in the pack, a commit that does not parse', async () => {
316 const from = await spooled(src.packs.ref);
317 const { bare, t } = repo();
318 const small = await copyTree(t, from, { budget: { ...BUDGET, maxObjectBytes: 1000 }, ref: REF, commit: src.commit });
319 assert.equal(small.error.code, 'limit_exceeded');
320 const otherTree = te.encode(`tree ${'1'.repeat(40)}\nauthor a <a@x> 0 +0000\ncommitter a <a@x> 0 +0000\n\nx\n`);
321 const noTree = planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: otherTree });
322 assert.equal(noTree.error.code, 'bad_request');
323 assert.match(noTree.error.message, /is not a tree in the pack/);
324 assert.equal(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: te.encode('not a commit') }).error.code, 'pack_corrupt');
325 assert.equal(refsOf(bare), '', 'nothing pushed');
326});
328test('refusals at a step and at status: an edited plan, another pack, other ids, other commit bytes, a step not in the plan, a ref taken', async () => {
329 const from = await spooled(src.packs.ref);
330 const other = await spooled(src.packs.ofs);
331 const plan = must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit }));
332 const { bare, t } = repo();
333 const edited = await transferStep(t, from, { ...plan, ref: 'refs/heads/other' }, 0, { commit: src.commit, ref: REF });
334 assert.match(edited.error.message, /does not match its hash/);
335 assert.match((await transferStep(t, other, plan, 0, { commit: src.commit, ref: REF })).error.message, /is not the planned one/);
336 const ids = Uint8Array.from(from.resolved.columns.oid);
337 ids.fill(0x11, 0, 20 * from.resolved.count);
338 const lyingOid = i => (Number.isInteger(i) && i >= 0 && i < from.resolved.count ? { tag: 'some', value: '11'.repeat(20) } : { tag: 'none' });
339 const lying = { ...from, resolved: { ...from.resolved, oid: lyingOid, columns: { ...from.resolved.columns, oid: ids } } };
340 assert.match((await transferStep(t, lying, plan, 0, { commit: src.commit, ref: REF })).error.message, /the pack is not the one planned/);
341 const last = plan.steps.length - 1;
342 const wrongCommit = te.encode(new TextDecoder().decode(src.commit).replace('two', 'three'));
343 assert.match((await transferStep(t, from, plan, last, { commit: wrongCommit, ref: REF })).error.message, /does not give the planned head/);
344 assert.equal((await transferStep(t, from, plan, last, { commit: te.encode('junk'), ref: REF })).error.code, 'pack_corrupt');
345 assert.match((await transferStep(t, from, plan, plan.steps.length, { commit: src.commit, ref: REF })).error.message, /is not in the plan/);
346 assert.equal(refsOf(bare), '', 'nothing pushed');
347 // a temp ref this plan never writes, then the target taken by another commit
348 git(work, ['push', '-q', bare, `${src.head}:${plan.tempRef}`]);
349 const foreign = await transferStatus(t, plan, { ref: REF });
350 assert.equal(foreign.error.code, 'cas_conflict');
351 git(bare, ['update-ref', '-d', plan.tempRef]);
352 git(work, ['push', '-q', bare, `${src.head}:${REF}`]);
353 const taken = await runTransfer(t, from, plan, src.commit, { ref: REF });
354 assert.equal(taken.error.code, 'cas_conflict');
355 assert.match(taken.error.message, /already exists/);
356});
358test('commit policy: signatures dropped (with their continuation lines), other headers and the encoding kept', () => {
359 const text = [
360 `tree ${src.tree}`, `parent ${src.head}`, 'author Ann <a@x> 1700000000 +0100', 'committer Cy <c@x> 1700000001 -0500',
361 'encoding ISO-8859-1', 'gpgsig -----BEGIN PGP SIGNATURE-----', ' line two', ' -----END PGP SIGNATURE-----',
362 'x-custom kept', ' continued', 'mergetag object 1', ' more', '', 'subject', '',
363 ].join('\n');
364 const made = must(rootCommit(te.encode(text))).commit;
365 const out = new TextDecoder().decode(made.data);
366 assert.equal(out, [`tree ${src.tree}`, 'author Ann <a@x> 1700000000 +0100', 'committer Cy <c@x> 1700000001 -0500', 'encoding ISO-8859-1', 'x-custom kept', ' continued', '', 'subject', ''].join('\n'));
367 assert.equal(made.oid, git(work, ['hash-object', '-t', 'commit', '--stdin'], { input: out }).trim());
368});
370test('temp trees: fanned out by the first two hex digits, ids git agrees with', async () => {
371 const from = await spooled(src.packs.ref);
372 const all = [...Array(from.pack.count).keys()].filter(i => from.resolved.columns.type[i] === 3);
373 const { subtrees, root: rootTree } = must(batchTrees(from.resolved, all));
374 const trees = [...subtrees, rootTree];
375 const scratch = makeBareRepo(root, `scratch${n++}`);
376 for (const o of trees) assert.equal(git(scratch, ['hash-object', '-w', '-t', 'tree', '--stdin'], { input: Buffer.from(o.data) }).trim(), o.oid);
377 const names = git(scratch, ['ls-tree', rootTree.oid]).split('\n').filter(Boolean).map(l => l.split('\t')[1]);
378 assert.ok(names.every(x => /^[0-9a-f]{2}$/.test(x)));
379 assert.equal(trees.length, names.length + 1);
380});
382test('sweepTransfers: removes an abandoned transfer\'s temp ref, keeps what it is told to', async () => {
383 const from = await spooled(src.packs.ref);
384 const plan = must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit }));
385 const { bare, t } = repo();
386 assert.deepEqual(must(await sweepTransfers(t)), []);
387 must(await runTransfer(t, from, plan, src.commit, { maxSteps: 2, ref: REF }));
388 assert.deepEqual(must(await sweepTransfers(t, { keep: [plan.tempRef] })), []);
389 assert.deepEqual(must(await sweepTransfers(t)), [plan.tempRef]);
390 assert.equal(git(bare, ['for-each-ref', 'refs/ply']).trim(), '');
391});
393// #298: a sweep deletes only refs ply's transfers create: names of the shape refs/ply/transfer/<40 hex key> (README
394// 'Resumable transfer'), so any other prefix is refused before a request and a near miss is never matched by text.
395test('sweepTransfers: a prefix outside refs/ply/transfer/ is bad_request before any request; only temp refs go', async () => {
396 const from = await spooled(src.packs.ref);
397 const plan = must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit }));
398 let requests = 0;
399 const { bare, t } = repo({ fetchImpl: (...a) => { requests++; return globalThis.fetch(...a); } });
400 must(await runTransfer(t, from, plan, src.commit, { maxSteps: 2, ref: REF }));
401 const tip = git(bare, ['rev-parse', plan.tempRef]).trim();
402 // refs no transfer made: a branch, a tag, a sibling namespace, a near-miss name, a deeper name under the prefix
403 const others = ['refs/heads/main', 'refs/tags/v1', 'refs/ply/transfer-old/x', 'refs/ply/transferx', `refs/ply/transfer/${'0'.repeat(39)}`, 'refs/ply/transfer/sub/x',
404 // 40 characters that are not a lowercase-hex oid (git's object names are lowercase hex): not a temp key
405 `refs/ply/transfer/${'A'.repeat(40)}`, `refs/ply/transfer/${'g'.repeat(40)}`];
406 for (const r of others) git(bare, ['update-ref', r, tip]);
407 const listing = () => git(bare, ['for-each-ref', '--format=%(refname)']).trim().split('\n').sort();
408 const before = listing();
409 requests = 0;
410 for (const prefix of ['', 'refs/', 'refs/heads/', 'refs/tags/', 'refs/ply/transfer', 'refs/ply/transfer-old/', 'refs/ply/', 'refs/ply/transfer/../', 'refs/heads/ply/transfer/', 'refs/ply/transfer/sub', 42, {},
411 // ends in refs/ply/transfer/ but does not start with it: a branch namespace, not ply's
412 'refs/heads/refs/ply/transfer/']) {
413 const r = await sweepTransfers(t, { prefix });
414 assert.equal(r.ok, false, `prefix ${JSON.stringify(prefix)} is refused`);
415 assert.equal(r.error.code, 'bad_request', JSON.stringify(prefix));
416 assert.equal(r.error.what, 'prefix', JSON.stringify(prefix));
417 }
418 // the reason is caller-visible: one prefix per arm (not a string; outside the root or without '/'; a name git would not push)
419 const why = async prefix => { const r = await sweepTransfers(t, { prefix }); assert.equal(r.ok, false); return r.error.message; };
420 assert.match(await why(42), /^sweepTransfers: prefix 42 is not a string$/);
421 assert.match(await why('refs/heads/'), /^sweepTransfers: prefix "refs\/heads\/" is not refs\/ply\/transfer\/ or a namespace ending in '\/' below it$/);
422 assert.match(await why('refs/ply/transfer/../'), /^sweepTransfers: prefix "refs\/ply\/transfer\/\.\.\/" is not a ref namespace git would push$/);
423 assert.equal(requests, 0, 'no refusal made a request');
424 assert.deepEqual(listing(), before, 'git: every ref is still there after the refusals');
425 assert.deepEqual(must(await sweepTransfers(t, { prefix: 'refs/ply/transfer/' })), [plan.tempRef]);
426 assert.deepEqual(listing(), before.filter(r => r !== plan.tempRef), 'git: only the temp ref is gone');
427});
429// #298: README 'Resumable transfer': prefix may be refs/ply/transfer/ "or a namespace below it ending in /".
430test('sweepTransfers: a nested namespace below refs/ply/transfer/ is swept; only its temp ref goes', async () => {
431 const tempPrefix = 'refs/ply/transfer/sub/';
432 const from = await spooled(src.packs.ref);
433 const plan = must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit, tempPrefix }));
434 assert.ok(plan.tempRef.startsWith(tempPrefix));
435 const { bare, t } = repo();
436 must(await runTransfer(t, from, plan, src.commit, { maxSteps: 2, ref: REF, tempPrefix }));
437 const tip = git(bare, ['rev-parse', plan.tempRef]).trim();
438 // a root-level temp-shaped ref and a branch: outside the nested namespace, so kept
439 for (const r of ['refs/heads/keep', `refs/ply/transfer/${'1'.repeat(40)}`]) git(bare, ['update-ref', r, tip]);
440 const listing = () => git(bare, ['for-each-ref', '--format=%(refname)']).trim().split('\n').sort();
441 const before = listing();
442 assert.ok(before.includes(plan.tempRef));
443 assert.deepEqual(must(await sweepTransfers(t, { prefix: tempPrefix })), [plan.tempRef]);
444 assert.deepEqual(listing(), before.filter(r => r !== plan.tempRef), 'git: only the nested temp ref is gone');
445});
447test('the plan: a tagged step per batch, prev an Option, refs git would push; a malformed plan read back is refused', async () => {
448 const from = await spooled(src.packs.ref);
449 const plan = must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit }));
450 const last = plan.steps.length - 1;
451 plan.steps.forEach((s, i) => {
452 assert.equal(s.kind, i === last ? 'final' : 'batch');
453 if (i === 0) assert.deepEqual(s.prev, { tag: 'none' });
454 else assert.deepEqual(s.prev, { tag: 'some', value: plan.steps[i - 1].temp });
455 if (s.kind === 'final') assert.equal('temp' in s, false);
456 else assert.match(s.temp, /^[0-9a-f]{40}$/);
457 });
458 // one push: the final step alone, no temp ref to delete
459 const one = must(planTransfer(from.pack, from.resolved, { budget: { ...BUDGET, maxRequestBytes: 10_000_000 }, ref: REF, commit: src.commit }));
460 assert.deepEqual(one.steps, [{ kind: 'final', prev: { tag: 'none' } }]);
461 for (const ref of ['main', 'refs/heads', 'refs/@', 'refs/heads/a..b', 7]) {
462 const r = planTransfer(from.pack, from.resolved, { budget: BUDGET, ref, commit: src.commit });
463 assert.equal(r.error.code, 'bad_request', String(ref));
464 assert.equal(r.error.what, 'ref');
465 }
466 for (const tempPrefix of ['refs/ply', 'refs/', 'nope/', 'refs/ply/transfer']) assert.equal(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit, tempPrefix }).error.what, 'tempPrefix', tempPrefix);
467 assert.ok(must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit, tempPrefix: 'refs/x/y/' })).tempRef.startsWith('refs/x/y/'));
468 // an index the resolved columns do not hold: refused, not read past the end
469 const outside = batchTrees(from.resolved, [from.resolved.count]);
470 assert.equal(outside.error.code, 'bad_request');
471 assert.match(outside.error.message, /is not in the resolved index/);
472 const longer = planTransfer({ ...from.pack, count: from.pack.count + 1 }, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit });
473 assert.match(longer.error.message, /entry \d+ is not in the resolved index/);
474 // read back: each malformed shape refused, even with its hash recomputed by hand
475 const json = JSON.parse(JSON.stringify(plan));
476 const bad = [
477 p => { p.steps[0] = { ...p.steps[0], prev: null }; },
478 p => { p.steps[0] = { ...p.steps[0], temp: null }; },
479 p => { p.steps[last] = { ...p.steps[last], temp: p.steps[0].temp }; },
480 p => { p.steps[0] = { ...p.steps[0], kind: 'final' }; },
481 p => { p.steps[last] = { ...p.steps[last], kind: 'batch' }; },
482 p => { p.steps.push({ kind: 'final', prev: { tag: 'none' } }); },
483 p => { p.steps = {}; },
484 p => { p.steps[1] = { ...p.steps[1], prev: { tag: 'some', value: 'x' } }; },
485 p => { p.head = 'HEAD'; },
486 p => { p.ref = 'main'; },
487 p => { p.tempRef = 'refs/@'; },
488 ];
489 for (const edit of bad) {
490 const p = structuredClone(json);
491 edit(p);
492 assert.equal(checkTransferPlan(p, { ref: REF }).ok, false, String(edit));
493 }
494 assert.ok(checkTransferPlan(json, { ref: REF }).ok);
495});
497test('ls-refs that lags a push: settled by reading again, not by sending again; a longer lag pauses; a resend refused after a lag goes on', async () => {
498 const from = await spooled(src.packs.ref);
499 const plan = must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit }));
500 // every push shows two reads late; three reads settle it: no step is sent twice
501 {
502 const { bare, t } = repo();
503 const lag = lagging(t, () => 2);
504 const run = must(await runTransfer(lag.t, from, plan, src.commit, { settle: { reads: 3, delayMs: 0 }, ref: REF }));
505 assert.deepEqual(run, { tag: 'done', head: plan.head, pushes: plan.steps.length, tempLeft: false });
506 assertCopied(bare, plan);
507 }
508 // a lag longer than the reads: the run pauses at the step it pushed ('settling'), the refs already past it
509 const { bare, t } = repo();
510 let lagNext = 4;
511 const lag = lagging(t, () => lagNext);
512 const paused = must(await runTransfer(lag.t, from, plan, src.commit, { settle: { reads: 2, delayMs: 0 }, ref: REF }));
513 assert.deepEqual(paused, { tag: 'paused', step: 0, pushes: 1, reason: 'settling', lastPush: 'acknowledged', lastStep: 'not_shown' });
514 assert.equal(git(bare, ['rev-parse', plan.tempRef]).trim(), plan.steps[0].temp, 'the push landed');
515 // the next run's first read still lags: it sends step 0 again, the server refuses it (stale), the refs settled show
516 // step 0 landed, and the run goes on to the end
517 lagNext = 0;
518 const before = lag.reads();
519 const resumed = must(await runTransfer(lag.t, from, plan, src.commit, { settle: { reads: 3, delayMs: 0 }, ref: REF }));
520 assert.equal(resumed.tag, 'done');
521 assert.equal(resumed.pushes, plan.steps.length, 'one resend, refused, then every other step once');
522 assert.ok(lag.reads() > before);
523 assertCopied(bare, plan);
524 // a refusal the refs do not explain stands: the target taken by another commit, read after the push
525 const other = repo();
526 git(work, ['push', '-q', other.bare, `${src.head}:${REF}`]);
527 assert.equal((await runTransfer(other.t, from, plan, src.commit, { ref: REF })).error.code, 'cas_conflict');
528});
530test('done reports a temp ref left behind; the spans: run > status, step, settle, each with the plan hash and an outcome', async () => {
531 const from = await spooled(src.packs.ref);
532 const plan = must(planTransfer(from.pack, from.resolved, { budget: BUDGET, ref: REF, commit: src.commit }));
533 const ex = arrayExporter();
534 const { bare, t } = repo({ tracer: new Tracer({ exporter: ex }) });
535 const run = must(await runTransfer(t, from, plan, src.commit, { settle: { reads: 1, delayMs: 0 }, ref: REF }));
536 assert.equal(run.tempLeft, false);
537 const spans = ex.lines.flatMap(flattenLine);
538 const named = name => spans.filter(s => s.name === name);
539 const [root] = named('git.transfer.run');
540 assert.equal(root.attrs['git.transfer.outcome'], 'done');
541 assert.equal(root.attrs['git.transfer.pushes'], plan.steps.length);
542 assert.equal(root.attrs['git.transfer.plan.hash'], plan.hash);
543 assert.equal(root.attrs['git.transfer.ref'], REF);
544 const steps = named('git.transfer.step');
545 assert.deepEqual(steps.map(s => s.attrs['git.transfer.step']), plan.steps.map((_, i) => i));
546 for (const [i, s] of steps.entries()) {
547 assert.equal(s.parentSpanId, root.spanId);
548 assert.equal(s.attrs['git.transfer.plan.hash'], plan.hash);
549 assert.equal(s.attrs['git.transfer.step.kind'], plan.steps[i].kind);
550 assert.equal(s.attrs['git.transfer.step.outcome'], 'landed');
551 assert.equal(s.attrs['git.transfer.batch.entries'], plan.pack.batches[i].entries.length);
552 assert.ok(s.attrs['git.transfer.batch.bytes'] > 0 && s.attrs['git.transfer.batch.bytes'] <= plan.pack.batches[i].packBytes + 300 * 68 + 32 * 1024);
553 assert.ok(spans.some(w => w.parentSpanId === s.spanId), 'the push\'s wire spans under the step');
554 }
555 const settles = named('git.transfer.settle');
556 assert.equal(settles.length, plan.steps.length);
557 assert.ok(settles.every(s => s.attrs['git.transfer.settle.outcome'] === 'moved' && s.parentSpanId === root.spanId));
558 assert.ok(named('git.transfer.status').every(s => s.attrs['git.transfer.plan.hash'] === plan.hash && typeof s.attrs['git.transfer.status.outcome'] === 'string'));
559 // a temp ref still on the destination after the ref was created (a host that kept it): done, tempLeft
560 git(bare, ['update-ref', plan.tempRef, plan.steps[0].temp]);
561 assert.deepEqual(must(await runTransfer(t, from, plan, src.commit, { ref: REF })), { tag: 'done', head: plan.head, pushes: 0, tempLeft: true });
562 assert.deepEqual(must(await sweepTransfers(t)), [plan.tempRef]);
563 const [sweep] = named('git.transfer.sweep').length ? named('git.transfer.sweep') : ex.lines.flatMap(flattenLine).filter(s => s.name === 'git.transfer.sweep');
564 assert.equal(sweep.attrs['git.transfer.sweep.deleted'], 1);
565});
567test('an exception out of resolved.oid() is the cause of a coded refusal: planTransfer, transferStep and runTransfer never reject, and the refs stay as they were', async () => {
568 const from = await spooled(src.packs.ofs);
569 const c = from.pack.columns;
570 const config = { budget: { ...BUDGET, maxObjects: 261 }, ref: REF, commit: src.commit, crossBatchRef: true };
571 const plan = must(planTransfer(from.pack, from.resolved, config));
572 // the first batch holding an OFS_DELTA whose stored base is in another batch: packSubset must ask for that base's id
573 const batchOf = new Map(plan.pack.batches.flatMap((b, k) => b.entries.map(i => [i, k])));
574 const isRewrite = (i, k) => c.storedType[i] === TYPE.OFS_DELTA && batchOf.get(c.baseIndex[i]) !== k;
575 const cutBatch = plan.pack.batches.findIndex((b, k) => b.entries.some(i => isRewrite(i, k)));
576 assert.ok(cutBatch > 0, 'the fixture cuts a chain after the first batch');
577 const cutBase = c.baseIndex[plan.pack.batches[cutBatch].entries.find(i => isRewrite(i, cutBatch))];
578 const baseBatch = batchOf.get(cutBase);
579 assert.ok(baseBatch < cutBatch);
580 // the expected cause is the test's own object: the refusal must carry this very Error, not a copy or its message
581 const marker = new Error('caller boom');
582 // every entry: reached building the temp tree. Only the cut base: at step `cutBatch`, reached only inside packSubset
583 const always = { ...from, resolved: { ...from.resolved, oid: () => { throw marker; } } };
584 const onlyBase = { ...from, resolved: { ...from.resolved, oid: i => { if (i === cutBase) throw marker; return from.resolved.oid(i); } } };
585 // a rejection (the exception escaping) fails the test at the await; a swallowed one has no cause
586 const refused = (r, label) => {
587 assert.equal(r.ok, false, label);
588 assert.equal(r.error.code, 'bad_request', label);
589 assert.equal(r.error.cause, marker, `${label}: the cause`);
590 };
591 refused(planTransfer(from.pack, always.resolved, config), 'planTransfer, oid() throws for every entry');
592 {
593 const { bare, t } = repo();
594 assert.equal(must(await runTransfer(t, from, plan, src.commit, { maxSteps: cutBatch, ref: REF })).tag, 'paused');
595 const before = refsOf(bare);
596 refused(await transferStep(t, onlyBase, plan, cutBatch, { commit: src.commit, ref: REF }), 'transferStep, oid() throws only for the cut base');
597 assert.equal(refsOf(bare), before, 'refs after transferStep, cut base');
598 refused(await transferStep(t, always, plan, cutBatch, { commit: src.commit, ref: REF }), 'transferStep, oid() throws for every entry');
599 assert.equal(refsOf(bare), before, 'refs after transferStep, every entry');
600 }
601 {
602 const { bare, t } = repo();
603 refused(await runTransfer(t, always, plan, src.commit, { ref: REF }), 'runTransfer, oid() throws for every entry');
604 assert.equal(refsOf(bare), '', 'no ref was created');
605 }
606 {
607 // the batch holding the cut base is the first to ask for its id: the refs are what an honest run paused there left
608 const honest = repo();
609 assert.equal(must(await runTransfer(honest.t, from, plan, src.commit, { maxSteps: baseBatch, ref: REF })).tag, 'paused');
610 const { bare, t } = repo();
611 refused(await runTransfer(t, onlyBase, plan, src.commit, { ref: REF }), 'runTransfer, oid() throws only for the cut base');
612 assert.equal(refsOf(bare), refsOf(honest.bare), 'refs after runTransfer, cut base');
613 }
614});
Diagnostics4 git calls · 407 ms
CallStatusSizeTime0407 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=400
git.response.bytes=38434
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=38434
git.pkt.payload.max=38339
git.sideband.band1.bytes=38338
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=63
git.pack.bytes=38338
git.pack.ofs_delta=0
git.pack.ref_delta=8
git.pack.chain.max=8
git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77
git.pack.full=55
Open pack map
fetch20011 KiB400 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=10776
git.response.cache=MISS
git.response.duration_ms=400
git.fetch.wants=0135b9348d8e2dcce1e648431dafbb2b4ef23d5b
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=10776
git.pkt.payload.max=10755
git.sideband.band1.bytes=10754
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e28a3e3aa2bd4a
Request
1315305b…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format