Artifactspublic-v0

ply

main
ply4 git calls318 ms41 KiB2 cachedprotocol v2
129 lines9.8 KiBRaw
1// Parity with git's own client. For each clone/fetch shape, git's CLI and our client issue the equivalent request;
2// we compare (1) the pack-objects invocation upload-pack made for each, (2) the shallow set git recorded vs our
3// shallow/unshallow lines, (3) the objects git stored vs the objects we received.
4import { test, beforeAll as before, afterAll as after } from 'vitest';
5import assert from 'node:assert/strict';
6import { join } from 'node:path';
7import { mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from 'node:fs';
8import { fileURLToPath } from 'node:url';
9import { git, gitBuf, tmpRoot, makeBareRepo, startGitServer } from './server.js';
10import { genDag, readModel } from './fuzz/dag.js';
11import { Rng } from './fuzz/prng.js';
12import { GitHttp } from '../src/client.js';
13import { Tracer } from '../src/otel.js';
14import { unwrap, unwrapAsync } from './unwrap.js';
16let root, cleanup, srv, hookLog, bare, model, dag, url, anc;
17before(async () => {
18 ({ root, cleanup } = tmpRoot());
19 const hookCfg = join(root, 'gitconfig'); hookLog = join(root, 'hooklog');
20 writeFileSync(hookCfg, `[uploadpack]\n\tpackObjectsHook = ${fileURLToPath(new URL('./fuzz/pack-objects-hook.sh', import.meta.url))}\n`);
21 srv = await startGitServer(root, { GIT_CONFIG_GLOBAL: hookCfg, PLY_HOOK_LOG: hookLog });
22 bare = makeBareRepo(root, 'p');
23 const work = join(root, 'w'); mkdirSync(work);
24 dag = genDag(new Rng(777), work, { commits: 18, branches: 3 });
25 git(work, ['push', '-q', '--all', bare]); git(work, ['push', '-q', '--tags', bare]);
26 model = readModel(bare);
27 url = `${srv.url}/p.git`;
28 fileUrl = `file://${bare}`;
29 cliEnv = { ...process.env, GIT_CONFIG_GLOBAL: join(root, 'gitconfig'), PLY_HOOK_LOG: hookLog };
30 // ancestors of main by generation (reachable commits only; dropped branch heads never reached the server)
31 anc = git(bare, ['rev-list', '--first-parent', 'main']).trim().split('\n');
32});
33after(async () => { await srv.close(); cleanup(); });
35const capture = () => ({ args: readFileSync(hookLog + '.args', 'utf8').trim().replace(/^git --shallow-file \S* pack-objects /, '').replace(/^git pack-objects /, ''), stdin: readFileSync(hookLog + '.stdin', 'utf8') });
36const norm = t => { const lines = t.trim().split('\n'); const i = lines.indexOf('--not'); const uniq = a => [...new Set(a)].sort(); return { shallow: uniq(lines.slice(0, i).filter(l => l.startsWith('--shallow'))), wants: uniq(lines.slice(0, i).filter(l => !l.startsWith('--shallow'))), not: uniq(lines.slice(i + 1).filter(Boolean)) }; };
37const objectsIn = dir => new Set(git(dir, ['cat-file', '--batch-check', '--batch-all-objects', '--unordered']).trim().split('\n').filter(Boolean).map(l => l.split(' ')[0]));
38const shallowOf = dir => existsSync(join(dir, '.git/shallow')) ? readFileSync(join(dir, '.git/shallow'), 'utf8').trim().split('\n').filter(Boolean).sort() : [];
39let n = 0;
40// git's own client speaks to the same upload-pack over the file:// smart transport (the sandbox blocks git's TCP to
41// local ports; the protocol, the server-side code path and the pack-objects hook are identical)
42let fileUrl, cliEnv;
43const cli = (args, cwd = root) => { rmSync(hookLog + '.args', { force: true }); git(cwd, ['-c', 'protocol.version=2', ...args.map(a => a === url ? fileUrl : a)], { env: cliEnv }); return capture(); };
45test('clone --depth N / --filter / --shallow-since / --shallow-exclude: same invocation, same shallow set, same objects', async () => {
46 const tip = dag.tip;
47 const c = unwrap(GitHttp.create(url, { tracer: new Tracer() }));
48 const scenarios = [
49 ['--depth=1', { wants: [tip], deepen: 1 }],
50 ['--depth=2', { wants: [tip], deepen: 2 }],
51 ['--depth=3', { wants: [tip], deepen: 3 }],
52 ['--depth=50', { wants: [tip], deepen: 50 }],
53 ['--depth=2 --filter=blob:none', { wants: [tip], deepen: 2, filter: 'blob:none' }],
54 ['--filter=blob:none', { wants: [tip], filter: 'blob:none' }],
55 ['--filter=blob:limit=100', { wants: [tip], filter: 'blob:limit=100' }],
56 ['--filter=tree:0', { wants: [tip], filter: 'tree:0' }],
57 ['--filter=combine:blob:none+tree:2', { wants: [tip], filter: 'combine:blob:none+tree:2' }],
58 [`--shallow-since=${model.commits.get(anc[Math.min(3, anc.length - 1)]).date}`, { wants: [tip], deepenSince: model.commits.get(anc[Math.min(3, anc.length - 1)]).date }],
59 ['--shallow-exclude=v1', { wants: [tip], deepenNot: ['refs/tags/v1'] }],
60 ];
61 for (const [flags, req] of scenarios) {
62 const dst = join(root, `clone${n++}`);
63 // git: single-branch clone of main so the want set is exactly [tip]
64 const gitInv = cli(['clone', '-q', '--no-tags', '--single-branch', '-b', 'main', ...(req.filter ? ['--no-checkout'] : []), ...flags.split(' '), url, dst]); // a checkout would lazily fetch filtered blobs: a second request
65 const gitObjects = objectsIn(dst), gitShallow = shallowOf(dst);
66 rmSync(hookLog + '.args', { force: true });
67 const r = await unwrapAsync(c.fetchPack({ ...req, ofsDelta: true, thinPack: true, includeTag: true, noProgress: true }));
68 const ourInv = capture();
69 assert.equal(ourInv.args, gitInv.args, `${flags}: argv`);
70 assert.deepEqual(norm(ourInv.stdin), norm(gitInv.stdin), `${flags}: stdin`);
71 assert.deepEqual([...r.shallow].sort(), gitShallow, `${flags}: shallow set`);
72 const ours = new Set(r.objects.keys());
73 assert.deepEqual([...ours].sort(), [...gitObjects].sort(), `${flags}: objects (git stored ${gitObjects.size}, we received ${ours.size})`);
74 console.log(` ${flags.padEnd(40)} ${ours.size} objects, shallow ${gitShallow.length}: identical`);
75 }
76});
78test('fetch after a shallow clone: --deepen (relative), --depth=N, --unshallow, --shallow-since, --shallow-exclude', async () => {
79 const tip = dag.tip;
80 const c = unwrap(GitHttp.create(url, { tracer: new Tracer() }));
81 for (const [flags, mk] of [
82 ['--deepen=1', s => ({ deepen: 1, deepenRelative: true })],
83 ['--deepen=2', s => ({ deepen: 2, deepenRelative: true })],
84 ['--depth=3', s => ({ deepen: 3 })],
85 ['--depth=1', s => ({ deepen: 1 })],
86 ['--unshallow', s => ({ deepen: 2147483647 })],
87 [`--shallow-since=${model.commits.get(anc[Math.min(5, anc.length - 1)]).date}`, s => ({ deepenSince: model.commits.get(anc[Math.min(5, anc.length - 1)]).date })],
88 ['--shallow-exclude=v1', s => ({ deepenNot: ['refs/tags/v1'] })],
89 ]) {
90 const dst = join(root, `clone${n++}`);
91 cli(['clone', '-q', '--no-tags', '--single-branch', '-b', 'main', '--depth=2', url, dst]);
92 const before = objectsIn(dst), shallowBefore = shallowOf(dst);
93 const gitInv = cli(['fetch', '-q', ...flags.split(' '), 'origin', 'main'], dst);
94 const gitNew = [...objectsIn(dst)].filter(o => !before.has(o)).sort(), gitShallow = shallowOf(dst);
95 rmSync(hookLog + '.args', { force: true });
96 // our equivalent: want tip, declare the same shallow set, have the tip (git's local ref)
97 // thin bases resolve from the same local store git's client has (the clone before the fetch)
98 const external = oid => (before.has(oid) ? { type: { blob: 3, tree: 2, commit: 1, tag: 4 }[git(dst, ['cat-file', '-t', oid]).trim()], data: new Uint8Array(gitBuf(dst, ['cat-file', git(dst, ['cat-file', '-t', oid]).trim(), oid])) } : undefined);
99 const r = await unwrapAsync(c.fetchPack({ wants: [tip], haves: [tip], shallows: shallowBefore, ...mk(shallowBefore), ofsDelta: true, thinPack: true }, { external }));
100 const ourInv = capture();
101 assert.equal(ourInv.args, gitInv.args, `${flags}: argv`);
102 assert.deepEqual(norm(ourInv.stdin), norm(gitInv.stdin), `${flags}: stdin`);
103 const expectedShallow = [...new Set([...shallowBefore.filter(s => !r.unshallow.includes(s)), ...r.shallow])].sort();
104 assert.deepEqual(expectedShallow, gitShallow, `${flags}: shallow set after`);
105 assert.deepEqual([...r.objects.keys()].sort(), gitNew, `${flags}: new objects (git +${gitNew.length}, we ${r.objects.size})`);
106 console.log(` ${flags.padEnd(40)} +${gitNew.length} objects, shallow ${shallowBefore.length}→${gitShallow.length}: identical`);
107 }
108});
110test('ls-refs == git ls-remote (prefixes, symrefs, peeled tags, many refs)', async () => {
111 for (let i = 0; i < 300; i++) git(bare, ['update-ref', `refs/heads/many/b${String(i).padStart(4, '0')}`, anc[i % anc.length]]);
112 git(bare, ['update-ref', 'refs/tags/light', dag.tip]);
113 git(bare, ['symbolic-ref', 'refs/heads/alias', 'refs/heads/main']);
114 const c = unwrap(GitHttp.create(url, { tracer: new Tracer() }));
115 for (const prefixes of [[], ['refs/heads/'], ['refs/tags/'], ['refs/heads/many/b00'], ['refs/heads/main', 'refs/tags/v1'], ['HEAD']]) {
116 const { refs, head } = await unwrapAsync(c.lsRefs(prefixes));
117 const remote = git(bare, ['ls-remote', '--symref', ...(prefixes.length ? ['.', ...prefixes.map(p => p.replace(/\/$/, '/*'))] : ['.'])]).trim().split('\n').filter(Boolean);
118 // ls-remote with globs approximates ref-prefix (prefix match); compare on the exact-prefix filter ourselves
119 const exp = git(bare, ['ls-remote', '--symref', '.']).trim().split('\n').filter(Boolean).filter(l => prefixes.length === 0 || prefixes.some(p => l.split(/\s+/)[1].startsWith(p) || (l.startsWith('ref:') && l.split(/\s+/)[2].startsWith(p))));
120 const expOids = new Map(exp.filter(l => !l.startsWith('ref:')).map(l => { const [oid, name] = l.split(/\s+/); return [name, oid]; }));
121 for (const [name, oid] of expOids) if (!name.endsWith('^{}')) assert.equal(refs.get(name)?.oid, oid, `${prefixes}: ${name}`);
122 for (const [name, oid] of expOids) if (name.endsWith('^{}')) assert.equal(refs.get(name.slice(0, -3))?.peeled, oid, `peeled ${name}`);
123 assert.equal(refs.size, [...expOids.keys()].filter(k => !k.endsWith('^{}')).length, `${prefixes}: count`);
124 if (prefixes.length === 0 || prefixes.includes('HEAD')) assert.equal(head, 'refs/heads/main');
125 if (prefixes.length === 0) assert.equal(refs.get('refs/heads/alias')?.['symref-target'], 'refs/heads/main');
126 void remote;
127 }
128 console.log(' ls-refs: 6 prefix sets vs ls-remote --symref, 300+ refs: identical');
129});
Diagnostics4 git calls · 318 ms
CallStatusSizeTime0318 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=312
git.response.bytes=38434
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=38434
git.pkt.payload.max=38339
git.sideband.band1.bytes=38338
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=63
git.pack.bytes=38338
git.pack.ofs_delta=0
git.pack.ref_delta=8
git.pack.chain.max=8
git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77
git.pack.full=55
Open pack map
fetch2003.3 KiB312 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3426
git.response.cache=MISS
git.response.duration_ms=312
git.fetch.wants=f969aa290380823e61ac71127514b5d21cfb0a78
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3426
git.pkt.payload.max=3405
git.sideband.band1.bytes=3404
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2c0dba94189da
Request
5ae2114b…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format