GitHubivarvong

ply

main
ply7 git calls36 ms26 KiB5 cachedprotocol v2
119 lines6.7 KiBRaw
1// Thin REF bases (#143). Every cell is a base git stored, a type the caller claims, and exact or flipped
2// bytes. An honest base matches `git index-pack --stdin --strict --fix-thin` and `git cat-file`. Any other
3// buffer is one `git hash-object` would not store under that oid, and resolvePack refuses it before applyDelta.
4import { test } from 'vitest';
5import assert from 'node:assert/strict';
6import { execFileSync } from 'node:child_process';
7import { mkdtempSync, rmSync } from 'node:fs';
8import { tmpdir } from 'node:os';
9import { join } from 'node:path';
10import { buildPack, parsePack, resolvePack, makeDelta } from '../../src/pack.js';
11import { hashObject } from '../../src/objects.js';
12import { cells } from './thin-base.space.js';
13import { unwrap } from '../unwrap.js';
15const ENV = {
16 ...Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith('GIT_'))),
17 GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null',
18 GIT_AUTHOR_NAME: 'a', GIT_AUTHOR_EMAIL: 'a@x', GIT_COMMITTER_NAME: 'a', GIT_COMMITTER_EMAIL: 'a@x',
19 GIT_AUTHOR_DATE: '1700000000 +0000', GIT_COMMITTER_DATE: '1700000000 +0000',
20};
22const git = (dir, args, input) => execFileSync('git', args, { cwd: dir, env: ENV, input, maxBuffer: 1 << 26 });
23const line = (dir, args, input) => git(dir, args, input).toString().trim();
25function gitHash(dir, typeName, bytes, literally) {
26 const args = ['hash-object', ...(literally ? ['--literally'] : []), '-t', typeName, '--stdin'];
27 try {
28 return { ok: true, oid: line(dir, args, bytes) };
29 } catch (e) {
30 return { ok: false, status: e.status, err: String(e.stderr ?? '') };
31 }
34function flip(bytes) {
35 const out = Uint8Array.from(bytes);
36 const i = out.length - 1;
37 out[i] = (out[i] ?? 0) ^ 1;
38 return out;
41function tagBytes(object, name, message) {
42 return Buffer.from(`object ${object}\ntype commit\ntag ${name}\ntagger a <a@x> 1700000000 +0000\n\n${message}\n`);
45test('thin REF base: every claim × byte form against git hash-object and index-pack --fix-thin', async () => {
46 const all = cells();
47 assert.equal(all.length, 4 * 8 * 2);
48 const dir = mkdtempSync(join(tmpdir(), 'ply-thin-base-'));
49 try {
50 git(dir, ['init', '-q', '-b', 'main']);
51 const blobBase = Buffer.from(`blob-base\n${'x'.repeat(64)}\n`);
52 const blobNext = Buffer.from(`blob-base\n${'y'.repeat(64)}\n`);
53 const blob0 = line(dir, ['hash-object', '-w', '-t', 'blob', '--stdin'], blobBase);
54 const blob1 = line(dir, ['hash-object', '-w', '-t', 'blob', '--stdin'], blobNext);
55 const tree0 = line(dir, ['mktree'], `100644 blob ${blob0}\tf.txt\n`);
56 const tree1 = line(dir, ['mktree'], `100644 blob ${blob1}\tf.txt\n`);
57 const commit0 = line(dir, ['commit-tree', tree0, '-m', 'base']);
58 const commit1 = line(dir, ['commit-tree', tree1, '-p', commit0, '-m', 'next']);
59 const tag0 = line(dir, ['mktag'], tagBytes(commit0, 'v0', 'annotated base'));
60 const tag1 = line(dir, ['mktag'], tagBytes(commit1, 'v1', 'annotated next'));
61 const made = {
62 blob: { oid: blob0, base: new Uint8Array(blobBase), target: new Uint8Array(blobNext), targetOid: blob1 },
63 tree: { oid: tree0, base: new Uint8Array(git(dir, ['cat-file', 'tree', tree0])), target: new Uint8Array(git(dir, ['cat-file', 'tree', tree1])), targetOid: tree1 },
64 commit: { oid: commit0, base: new Uint8Array(git(dir, ['cat-file', 'commit', commit0])), target: new Uint8Array(git(dir, ['cat-file', 'commit', commit1])), targetOid: commit1 },
65 tag: { oid: tag0, base: new Uint8Array(git(dir, ['cat-file', 'tag', tag0])), target: new Uint8Array(git(dir, ['cat-file', 'tag', tag1])), targetOid: tag1 },
66 };
67 const cType = tag => ({ blob: 3, tree: 2, commit: 1, tag: 4 })[tag];
68 const packs = {};
69 for (const [tag, obj] of Object.entries(made)) {
70 assert.equal(hashObject(tag, obj.base), obj.oid, `${tag} base hashes as git stored it`);
71 const { pack } = await buildPack([{ type: cType(tag), data: makeDelta(obj.base, obj.target).delta, base: { tag: 'ref', oid: obj.oid } }]);
72 packs[tag] = unwrap(parsePack(pack));
73 const indexed = git(dir, ['index-pack', '--stdin', '--strict', '--fix-thin'], pack);
74 assert.match(indexed.toString(), /^pack\t[0-9a-f]{40}\n$/);
75 assert.equal(line(dir, ['cat-file', '-t', obj.targetOid]), tag);
76 assert.deepEqual(new Uint8Array(git(dir, ['cat-file', tag, obj.targetOid])), obj.target);
77 }
78 for (const c of all) {
79 const obj = made[c.base.tag];
80 const data = c.bytes.tag === 'exact' ? obj.base : flip(obj.base);
81 const parsed = packs[c.base.tag];
82 let threw = null;
83 let resolved = null;
84 try { resolved = unwrap(resolvePack(parsed, oid => (oid === obj.oid ? { type: c.claim.type, data } : undefined))); }
85 catch (e) { threw = e; }
86 if (c.honest) {
87 assert.equal(threw, null, `${c.id}: ${threw?.message}`);
88 const got = resolved.objects.get(obj.targetOid);
89 assert.ok(got, `${c.id}: result oid`);
90 assert.equal(got.type, c.base.type, `${c.id}: resolved type`);
91 assert.deepEqual(got.data, obj.target, `${c.id}: cat-file bytes`);
92 const gitOid = gitHash(dir, c.base.name, data, true);
93 assert.equal(gitOid.ok && gitOid.oid, obj.oid, `${c.id}: git hash-object --literally`);
94 continue;
95 }
96 assert.equal(threw?.code, 'pack_corrupt', `${c.id}: ${threw?.code} ${threw?.message}`);
97 assert.equal(threw.message.startsWith('delta:'), false, `${c.id}: refused before applyDelta (${threw.message})`);
98 if (c.claim.stored) {
99 const gitOid = gitHash(dir, c.claim.name, data, true);
100 assert.equal(gitOid.ok, true, `${c.id}: ${gitOid.err}`);
101 assert.notEqual(gitOid.oid, obj.oid, `${c.id}: git would store this under the base oid`);
102 assert.equal(threw.message, `pack: ref delta base ${obj.oid} hashes to ${gitOid.oid}`, c.id);
103 } else {
104 const refused = gitHash(dir, c.claim.name, data, false);
105 assert.equal(refused.ok, false, `${c.id}: git hash-object -t ${c.claim.name} should refuse`);
106 assert.match(refused.err, /invalid object type/, c.id);
107 assert.equal(threw.message, `pack: ref delta base ${obj.oid} has type ${c.claim.type}`, c.id);
108 }
109 }
110 // without the real base, git does not apply the thin pack
111 const empty = mkdtempSync(join(tmpdir(), 'ply-thin-empty-'));
112 try {
113 git(empty, ['init', '-q', '--bare']);
114 const pack = (await buildPack([{ type: 3, data: makeDelta(made.blob.base, made.blob.target).delta, base: { tag: 'ref', oid: made.blob.oid } }])).pack;
115 assert.throws(() => git(empty, ['index-pack', '--stdin', '--strict', '--fix-thin'], pack));
116 } finally { rmSync(empty, { recursive: true, force: true }); }
117 } finally { rmSync(dir, { recursive: true, force: true }); }
118});
Diagnostics7 git calls · 36 ms
CallStatusSizeTime036 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=11990
git.response.cache=UPDATING
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetchtree:02001.4 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=1428
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=tree:0
git.fetch.wants=74d7bb09987bbe78a9c0ea8f43a1b5a4e079f538
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1428
git.pkt.payload.max=1401
git.sideband.band1.bytes=1401
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1401
git.pack.trailer=65ea1a3aeef333a80a2dfa3f441e629d90497e8e
Open pack map
fetch2002.4 KiB · hit23 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=23
git.response.bytes=2407
git.response.cache=HIT
git.response.duration_ms=23
git.fetch.wants=e0ffadce6921cd0b5365c96a784074a2d70a74a2
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2407
git.pkt.payload.max=2380
git.sideband.band1.bytes=2380
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e14d24feb5a140
Request
deb7b01c…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format