main
ply4 git calls305 ms45 KiB2 cachedprotocol v2
287 lines13 KiBRaw
1// A scenario's seed objects built in memory, byte for byte as ply's probe runner builds them with git plumbing
2// (test/probes/runner.mjs buildSeed: fixed identity and dates, `git commit-tree -m <name>`, `git mktag`), so a plan run
3// from a Worker holds the same object ids as every run of it in ply's suite. test/bench-seed.test.ts checks that against
4// git itself. Pure apart from SHA-256 (the noise stream), which is the platform's crypto.subtle.
5import { Sha1, hashObject, parseOid, toHex, treeSort, encodeTree, type Oid } from 'ply';
6import type { Content, ObjRef, ObjectSpec, OidExpr, Push, Scenario, Seed } from 'ply/probes';
7import { mapGet } from '../absent.ts';
8import { type Result, err, ok } from './result.ts';
10export const ZERO_OID = '0'.repeat(40) as Oid;
11const IDENT = 'ply probe <probe@ply.invalid> 1700000000 +0000';
12const SHA256_BYTES = 32;
13const MODE_FILE = '100644';
14const MODE_DIR = '40000';
15const enc = new TextEncoder();
17export type ObjType = 'commit' | 'tree' | 'blob' | 'tag';
18export type StoredObject = { readonly oid: Oid; readonly type: ObjType; readonly data: Uint8Array; readonly links: readonly Oid[] };
19export type SeedStore = {
20 readonly names: ReadonlyMap<string, Oid>; // seed name -> oid
21 readonly byOid: Readonly<Record<string, string>>; // oid -> seed name (ply/probes Names)
22 readonly objects: ReadonlyMap<string, StoredObject>;
23 readonly trees: ReadonlyMap<string, Oid>; // commit name -> its root tree
24};
26// The deterministic incompressible bytes of a noise file: SHA-256 of `${seed}:${i}` for i = 0, 1, 2, ...
27export async function noiseBytes(seed: string, n: number): Promise<Uint8Array> {
28 const out = new Uint8Array(n);
29 const blocks = Math.ceil(n / SHA256_BYTES);
30 const digests = await Promise.all(Array.from({ length: blocks }, (_, i) => crypto.subtle.digest('SHA-256', enc.encode(`${seed}:${i}`))));
31 for (const [i, d] of digests.entries()) out.set(new Uint8Array(d).subarray(0, Math.min(SHA256_BYTES, n - i * SHA256_BYTES)), i * SHA256_BYTES);
32 return out;
34export async function contentBytes(c: Content): Promise<Uint8Array> {
35 switch (c.tag) {
36 case 'text': return enc.encode(c.text);
37 case 'noise': return noiseBytes(c.seed, c.bytes);
38 case 'repeat': return new Uint8Array(c.bytes).fill(c.byte);
39 }
41const contentSize = (c: Content): number => (c.tag === 'text' ? c.text.length : c.bytes);
42// The bytes a seed asks for, so a runner can refuse a plan too large for its memory before building it.
43export function seedBytes(seed: Seed): number {
44 return seed.objects.reduce((sum, o) => sum + (o.tag === 'commit' ? o.files.reduce((s, f) => s + contentSize(f.content), 0) : 0), 0);
47type Dir = { readonly files: Map<string, Oid>; readonly dirs: Map<string, Dir> };
48const emptyDir = (): Dir => ({ files: new Map(), dirs: new Map() });
50function put(objects: Map<string, StoredObject>, type: ObjType, data: Uint8Array, links: readonly Oid[]): Oid {
51 const oid = hashObject(type, data);
52 if (!objects.has(oid)) objects.set(oid, { oid, type, data, links });
53 return oid;
56function writeDir(objects: Map<string, StoredObject>, dir: Dir): Result<Oid, string> {
57 const entries: Array<{ mode: string; name: string; oid: string }> = [];
58 for (const [name, oid] of dir.files) entries.push({ mode: MODE_FILE, name, oid });
59 for (const [name, sub] of dir.dirs) {
60 const t = writeDir(objects, sub);
61 if (!t.ok) return t;
62 entries.push({ mode: MODE_DIR, name, oid: t.value });
63 }
64 const sorted = treeSort(entries);
65 if (!sorted.ok) return err(`tree sort: ${sorted.error.message}`);
66 const data = encodeTree(sorted.value);
67 if (!data.ok) return err(`tree encode: ${data.error.message}`);
68 return ok(put(objects, 'tree', data.value, sorted.value.map(e => e.oid as Oid)));
71// The tree of a commit as a nested map: its first parent's files, then its own written over them.
72function treeOf(objects: ReadonlyMap<string, StoredObject>, root: Oid): Dir {
73 const dir = emptyDir();
74 const t = mapGet(objects, root);
75 if (t.tag === 'none') return dir;
76 for (const e of parseTreeEntries(t.value.data)) {
77 if (e.mode === MODE_DIR) dir.dirs.set(e.name, treeOf(objects, e.oid));
78 else dir.files.set(e.name, e.oid);
79 }
80 return dir;
82export type TreeRow = { readonly mode: string; readonly name: string; readonly oid: Oid };
83export function parseTreeEntries(data: Uint8Array): TreeRow[] {
84 const rows: TreeRow[] = [];
85 const dec = new TextDecoder();
86 let at = 0;
87 while (at < data.length) {
88 const sp = data.indexOf(0x20, at);
89 const nul = data.indexOf(0, sp);
90 if (sp < 0 || nul < 0 || nul + 21 > data.length) break;
91 rows.push({ mode: dec.decode(data.subarray(at, sp)), name: dec.decode(data.subarray(sp + 1, nul)), oid: toHex(data.subarray(nul + 1, nul + 21)) as Oid });
92 at = nul + 21;
93 }
94 return rows;
97function placeFile(root: Dir, path: string, oid: Oid): void {
98 const parts = path.split('/');
99 const last = parts.pop();
100 let dir = root;
101 for (const p of parts) {
102 const next = mapGet(dir.dirs, p);
103 const sub = next.tag === 'some' ? next.value : emptyDir();
104 if (next.tag === 'none') dir.dirs.set(p, sub);
105 dir = sub;
106 }
107 dir.files.set(String(last), oid);
110async function buildCommit(objects: Map<string, StoredObject>, names: Map<string, Oid>, trees: Map<string, Oid>, o: Extract<ObjectSpec, { readonly tag: 'commit' }>): Promise<Result<Oid, string>> {
111 const parents: Oid[] = [];
112 for (const p of o.parents) {
113 const id = mapGet(names, p);
114 if (id.tag === 'none') return err(`commit ${o.name}: parent ${p} is not defined before it`);
115 parents.push(id.value);
116 }
117 const first = o.parents.length > 0 ? mapGet(trees, String(o.parents[0])) : mapGet(trees, '');
118 const root = first.tag === 'some' ? treeOf(objects, first.value) : emptyDir();
119 for (const f of o.files) placeFile(root, f.path, put(objects, 'blob', await contentBytes(f.content), []));
120 const tree = writeDir(objects, root);
121 if (!tree.ok) return tree;
122 trees.set(o.name, tree.value);
123 const text = `tree ${tree.value}\n${parents.map(p => `parent ${p}\n`).join('')}author ${IDENT}\ncommitter ${IDENT}\n\n${o.message}\n`;
124 return ok(put(objects, 'commit', enc.encode(text), [tree.value, ...parents]));
127// A raw tree's bytes, `<mode> <name>\0<20-byte id>` per entry in the plan's order (ply/probes RawTreeSpec), and the
128// blobs it names.
129function rawTree(store: SeedStore, o: Extract<ObjectSpec, { readonly tag: 'raw-tree' }>): Result<{ readonly data: Uint8Array; readonly links: readonly Oid[] }, string> {
130 const parts: Uint8Array[] = [];
131 const links: Oid[] = [];
132 for (const e of o.entries) {
133 const blob = resolveObjRef(store, e.blob);
134 if (!blob.ok) return err(`raw tree ${o.name}: ${blob.error}`);
135 parts.push(enc.encode(`${e.mode} ${e.name}\0`), Uint8Array.from(blob.value.match(/../g) || [], h => parseInt(h, 16)));
136 links.push(blob.value);
137 }
138 const data = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
139 let at = 0;
140 for (const p of parts) { data.set(p, at); at += p.length; }
141 return ok({ data, links });
144export async function buildSeed(seed: Seed): Promise<Result<SeedStore, string>> {
145 const objects = new Map<string, StoredObject>();
146 const names = new Map<string, Oid>();
147 const trees = new Map<string, Oid>();
148 for (const o of seed.objects) {
149 switch (o.tag) {
150 case 'commit': {
151 const c = await buildCommit(objects, names, trees, o);
152 if (!c.ok) return c;
153 names.set(o.name, c.value);
154 break;
155 }
156 case 'annotated-tag': {
157 const target = mapGet(names, o.target);
158 if (target.tag === 'none') return err(`tag ${o.name}: target ${o.target} is not defined before it`);
159 const body = `object ${target.value}\ntype commit\ntag ${o.tagName}\ntagger ${IDENT}\n\n${o.message}\n`;
160 names.set(o.name, put(objects, 'tag', enc.encode(body), [target.value]));
161 break;
162 }
163 case 'raw-tree': {
164 const t = rawTree({ names, byOid: {}, objects, trees }, o);
165 if (!t.ok) return t;
166 names.set(o.name, put(objects, 'tree', t.value.data, t.value.links));
167 break;
168 }
169 case 'raw-commit': {
170 const fromCommit = mapGet(trees, o.tree);
171 const tree = fromCommit.tag === 'some' ? fromCommit : mapGet(names, o.tree);
172 if (tree.tag === 'none') return err(`raw commit ${o.name}: tree ${o.tree} is not defined before it`);
173 const parents: Oid[] = [];
174 for (const p of o.parents) {
175 const id = mapGet(names, p);
176 if (id.tag === 'none') return err(`raw commit ${o.name}: parent ${p} is not defined before it`);
177 parents.push(id.value);
178 }
179 const text = `tree ${tree.value}\n${parents.map(p => `parent ${p}\n`).join('')}author ${o.author}\ncommitter ${IDENT}\n\n${o.message}\n`;
180 names.set(o.name, put(objects, 'commit', enc.encode(text), [tree.value, ...parents]));
181 break;
182 }
183 case 'raw-tag': {
184 // the type line as written, which may disagree with the target or name a missing one
185 const target = mapGet(names, o.object);
186 if (target.tag === 'none') return err(`raw tag ${o.name}: object ${o.object} is not defined before it`);
187 const body = `object ${target.value}\ntype ${o.type}\ntag ${o.tagName}\ntagger ${IDENT}\n\n${o.message}\n`;
188 names.set(o.name, put(objects, 'tag', enc.encode(body), [target.value]));
189 break;
190 }
191 case 'phantom':
192 // an id nothing holds: named, never stored
193 names.set(o.name, toHex(new Sha1().update(enc.encode(`probe phantom ${o.seed}`)).digest()) as Oid);
194 break;
195 }
196 }
197 const byOid: Record<string, string> = {};
198 for (const [n, oid] of names) byOid[oid] = n;
199 return ok({ names, byOid, objects, trees });
202// ---- reading a plan against a built seed ----
203export function oidOf(store: SeedStore, e: OidExpr): Result<Oid, string> {
204 if (e.tag === 'zero') return ok(ZERO_OID);
205 const id = mapGet(store.names, e.name);
206 return id.tag === 'some' ? ok(id.value) : err(`no seed object ${e.name}`);
208export function resolveObjRef(store: SeedStore, r: ObjRef): Result<Oid, string> {
209 if (r.tag === 'commit') return oidOf(store, { tag: 'obj', name: r.name });
210 const tree = mapGet(store.trees, r.of);
211 if (tree.tag === 'none') return err(`no tree for ${r.of}`);
212 let at: Oid = tree.value;
213 for (const part of r.path.split('/')) {
214 const t = mapGet(store.objects, at);
215 if (t.tag === 'none') return err(`tree ${at} not in the seed`);
216 const row = parseTreeEntries(t.value.data).find(e => e.name === part);
217 if (typeof row !== 'object') return err(`${r.of}:${r.path}: no entry ${part}`);
218 at = row.oid;
219 }
220 return ok(at);
223// Every object reachable from `tips` (commits, their trees and parents, tags and their targets), in walk order.
224export function reachable(store: SeedStore, tips: readonly Oid[]): Oid[] {
225 const seen = new Set<string>();
226 const order: Oid[] = [];
227 const stack = [...tips].reverse();
228 while (stack.length > 0) {
229 const oid = stack.pop();
230 if (typeof oid !== 'string' || seen.has(oid) || oid === ZERO_OID) continue;
231 seen.add(oid);
232 const o = mapGet(store.objects, oid);
233 if (o.tag === 'none') continue;
234 order.push(oid);
235 for (const l of [...o.value.links].reverse()) stack.push(l);
236 }
237 return order;
240// The objects a push carries: reachable from its new ids and not from `have` (the ids the target holds), minus the
241// plan's `omit` (test/probes/runner.mjs objectsFor); none for a `no-objects` defect pack; exactly the named objects for a
242// `listed` pack, with no walk.
243export function objectsFor(store: SeedStore, push: Push, have: readonly Oid[]): Result<StoredObject[], string> {
244 if (push.pack.tag === 'defect' && push.pack.defect.tag === 'no-objects') return ok([]);
245 if (push.pack.tag === 'listed') {
246 const listed: StoredObject[] = [];
247 for (const n of push.pack.objects) {
248 const id = oidOf(store, { tag: 'obj', name: n });
249 if (!id.ok) return id;
250 const o = mapGet(store.objects, id.value);
251 if (o.tag === 'none') return err(`listed object ${n} is not stored in the seed`);
252 listed.push(o.value);
253 }
254 return ok(listed);
255 }
256 const news: Oid[] = [];
257 for (const u of push.updates) {
258 const n = oidOf(store, u.new);
259 if (!n.ok) return n;
260 if (n.value !== ZERO_OID) news.push(n.value);
261 }
262 const omit = new Set<string>();
263 for (const r of push.omit) {
264 const id = resolveObjRef(store, r);
265 if (!id.ok) return id;
266 omit.add(id.value);
267 }
268 const held = new Set<string>(reachable(store, have));
269 const out: StoredObject[] = [];
270 for (const oid of reachable(store, news)) {
271 const o = mapGet(store.objects, oid);
272 if (o.tag === 'some' && !held.has(oid) && !omit.has(oid)) out.push(o.value);
273 }
274 return ok(out);
277export function seedTips(store: SeedStore, s: Scenario): Result<Oid[], string> {
278 const tips: Oid[] = [];
279 for (const r of s.seed.refs) {
280 const id = oidOf(store, { tag: 'obj', name: r.at });
281 if (!id.ok) return id;
282 tips.push(id.value);
283 }
284 return ok(tips);
287export const isOid = (s: string): s is Oid => parseOid(s).ok;
Diagnostics4 git calls · 305 ms
CallStatusSizeTime0305 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2004.1 KiB300 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=300
git.response.bytes=4247
git.response.cache=MISS
git.response.duration_ms=300
git.fetch.wants=3b69a797bf21183b02a2ed2c7a847d0fbf17961a
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4247
git.pkt.payload.max=4226
git.sideband.band1.bytes=4225
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e31c379dfd8682
Request
2c53bd2a…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format