GitHubivarvong
ply
main
1// The objects a pack carries, walked from tips, over any object store: what pushRepo sends, and what a repack or an
2// export writes. Workers-safe (no node:); pushRepo (repo-node.ts) reads its store from `.git`, a Worker from a fetched
3// pack (`mapStore` over resolvePack's objects).
4//
5// objectsForPack(store, tips, have) → everything reachable from `tips` that `have` lacks, in pack order: commits
6// parents first, then tags, then each commit's new trees and blobs depth first in
7// tree order, each with the path it first appears at (planDeltas' candidates)
8// reachableObjects(store, tips) → everything reachable from `tips` that the store has (a remote's side of a push:
9// blobs are marked from tree entries and never read; a missing object is skipped)
10//
11// Gitlinks are not followed. A wanted object the store lacks is `bad_request` (`what: 'object'`); an object of the wrong
12// type where a tree, commit or blob is named is `pack_corrupt`. Every commit, tree and tag is parsed under `limits`.
13/* oxlint-disable no-await-in-loop -- the walks read one object at a time from the store, in pack order (the order is the output) */
14import { fail } from './errors.js';
15import type { PlyError } from './errors.js';
16import { DEFAULT_LIMITS, parseLimits } from './limits.js';
17import type { CheckedLimits, LimitsInput } from './limits.js';
18import { parseCommit, parseTag, parseTree, TYPE, TYPE_ID } from './objects.js';
19import { MODE_COMMIT, MODE_TREE } from './format.js';
20import type { DeltaCandidate } from './delta.js';
21import { fromNullable, mapGet, readKey } from './absent.js';
22import { getOrElse, ok } from './result.js';
23import type { Option, Result } from './result.js';
25/** An object as a store returns it: its type id (TYPE) and its bytes. */
26export type StoredObject = { readonly type: number; readonly data: Uint8Array };
27/** Where the walk reads objects: `get` answers the object, or null when the store does not have it. */
28export type ObjectStore = { readonly get: (oid: string) => Promise<Result<StoredObject | null, PlyError>> };
29/** A pack candidate with its id: planDeltas reads type, data and path; the id is for the caller's index. */
30export type PackCandidate = DeltaCandidate & { readonly oid: string };
32/** A store over objects already in memory (resolvePack's map, or any oid → { type, data } map). */
33export function mapStore(objects: ReadonlyMap<string, StoredObject>): ObjectStore {
34 // oxlint-disable-next-line typescript/promise-function-async -- a settled answer, no work to await
35 return { get: (oid: string) => { const hit = mapGet(objects, oid); return Promise.resolve(ok(hit.tag === 'some' ? hit.value : null)); } };
36}
38// The next item of a work stack (none when it is empty).
39export const popped = <T>(stack: T[]): Option<T> => (stack.length === 0 ? { tag: 'none' } : fromNullable(stack.pop()));
40const TYPE_IDS: Readonly<Record<string, number>> = TYPE_ID;
41// `source` names the store in a missing-object error ("missing from <source>").
42export type Walk = { readonly store: ObjectStore; readonly limits: CheckedLimits; readonly source: string };
43const TYPE_NAMES: Readonly<Record<number, string>> = { [TYPE.COMMIT]: 'commit', [TYPE.TREE]: 'tree', [TYPE.BLOB]: 'blob', [TYPE.TAG]: 'tag' };
44const typeName = (type: number): string => getOrElse(readKey(TYPE_NAMES, type), String(type));
45const wrongType = (oid: string, type: number, want: number): Result<never, PlyError> =>
46 fail('pack_corrupt', `${oid} is a ${typeName(type)}, not a ${typeName(want)}`, { what: 'object', path: oid });
48export type Child = { readonly oid: string; readonly type: number; readonly name: string };
49// The objects one object names: a commit's tree and parents, a tree's entries (not gitlinks), a tag's object.
50export function children(o: StoredObject, limits: CheckedLimits): Result<Child[], PlyError> {
51 switch (o.type) {
52 case TYPE.COMMIT: {
53 const c = parseCommit(o.data, limits);
54 if (!c.ok) return c;
55 return ok([...c.value.parents.map(p => ({ oid: p, type: TYPE.COMMIT, name: '' })), { oid: c.value.tree, type: TYPE.TREE, name: '' }]);
56 }
57 case TYPE.TREE: {
58 const t = parseTree(o.data, limits);
59 if (!t.ok) return t;
60 return ok(t.value.filter(e => e.mode !== MODE_COMMIT).map(e => ({ oid: e.oid, type: e.mode === MODE_TREE ? TYPE.TREE : TYPE.BLOB, name: e.name })));
61 }
62 case TYPE.TAG: {
63 const t = parseTag(o.data);
64 if (!t.ok) return t;
65 return ok([{ oid: t.value.object, type: getOrElse(readKey(TYPE_IDS, t.value.type), TYPE.COMMIT), name: '' }]);
66 }
67 default: return ok([]);
68 }
69}
71// An object the store must have (the pack carries it, or the walk decides with it).
72async function need(w: Walk, oid: string, type: number | null, where: string): Promise<Result<StoredObject, PlyError>> {
73 const got = await w.store.get(oid);
74 if (!got.ok) return got;
75 if (got.value === null) return fail('bad_request', `${oid} (${where}) is missing from ${w.source}`, { what: 'object', path: oid });
76 if (type !== null && got.value.type !== type) return wrongType(oid, got.value.type, type);
77 return ok(got.value);
78}
80// Everything reachable from `tips` that the store has (for a push: what the remote has). Blobs are marked from their
81// tree entries and never read; an object the local repository lacks is skipped (the remote has it, ply does not).
82export async function reachable(w: Walk, tips: readonly string[]): Promise<Result<Set<string>, PlyError>> {
83 const seen = new Set<string>();
84 const stack = [...tips];
85 for (let next = popped(stack); next.tag === 'some'; next = popped(stack)) {
86 const oid = next.value;
87 if (seen.has(oid)) continue;
88 const got = await w.store.get(oid);
89 if (!got.ok) return got;
90 if (got.value === null) continue;
91 seen.add(oid);
92 const kids = children(got.value, w.limits);
93 if (!kids.ok) return kids;
94 for (const k of kids.value) if (k.type === TYPE.BLOB) seen.add(k.oid); else stack.push(k.oid);
95 }
96 return ok(seen);
97}
98// What `have` lacks, in pack order: commits parents-first, then tags, then each commit's new trees and blobs in
99// tree order with their paths (delta candidates: the previous version of a path is earlier in the pack).
100export async function missing(w: Walk, tips: readonly string[], have: ReadonlySet<string>): Promise<Result<PackCandidate[], PlyError>> {
101 const seen = new Set<string>(have);
102 const commits: Array<{ readonly oid: string; readonly tree: string; readonly data: Uint8Array }> = [];
103 const tags: PackCandidate[] = [];
104 const rootObjects: Array<{ readonly oid: string; readonly type: number }> = [];
105 // tips: peel tags (sent), then commits in post-order (parents first) by an explicit stack
106 const stack: Array<{ readonly oid: string; readonly done: boolean }> = tips.map(oid => ({ oid, done: false })).toReversed();
107 const parsed = new Map<string, { readonly tree: string; readonly data: Uint8Array }>();
108 for (let next = popped(stack); next.tag === 'some'; next = popped(stack)) {
109 const { oid, done } = next.value;
110 if (done) { const c = mapGet(parsed, oid); if (c.tag === 'some') commits.push({ oid, ...c.value }); continue; }
111 if (seen.has(oid)) continue;
112 seen.add(oid);
113 const got = await need(w, oid, null, 'a tip or an ancestor');
114 if (!got.ok) return got;
115 if (got.value.type === TYPE.TAG) {
116 tags.push({ oid, type: TYPE.TAG, data: got.value.data, path: null });
117 const kids = children(got.value, w.limits);
118 if (!kids.ok) return kids;
119 for (const k of kids.value) stack.push({ oid: k.oid, done: false });
120 continue;
121 }
122 if (got.value.type !== TYPE.COMMIT) { seen.delete(oid); rootObjects.push({ oid, type: got.value.type }); continue; }
123 const c = parseCommit(got.value.data, w.limits);
124 if (!c.ok) return c;
125 parsed.set(oid, { tree: c.value.tree, data: got.value.data });
126 stack.push({ oid, done: true });
127 for (const p of c.value.parents.toReversed()) stack.push({ oid: p, done: false });
128 }
129 const out: PackCandidate[] = [...commits.map(c => ({ oid: c.oid, type: TYPE.COMMIT, data: c.data, path: null })), ...tags];
130 // trees and blobs, depth first in tree order, each with the path it first appears at
131 const walkTree = async (oid: string, type: number, path: string): Promise<Result<null, PlyError>> => {
132 if (seen.has(oid)) return ok(null);
133 seen.add(oid);
134 const got = await need(w, oid, type, path === '' ? 'a root tree' : path);
135 if (!got.ok) return got;
136 out.push({ oid, type, data: got.value.data, path });
137 if (type !== TYPE.TREE) return ok(null);
138 const kids = children(got.value, w.limits);
139 if (!kids.ok) return kids;
140 for (const k of kids.value) {
141 const r = await walkTree(k.oid, k.type, path === '' ? k.name : `${path}/${k.name}`);
142 if (!r.ok) return r;
143 }
144 return ok(null);
145 };
146 for (const c of commits) {
147 const r = await walkTree(c.tree, TYPE.TREE, '');
148 if (!r.ok) return r;
149 }
150 // a tag of a tree or a blob: that object and (for a tree) what it names
151 for (const o of rootObjects) {
152 const r = await walkTree(o.oid, o.type, '');
153 if (!r.ok) return r;
154 }
155 return ok(out);
156}
158/** Everything reachable from `tips` that `store` has. Blobs are marked from their tree entries and never read. */
159// oxlint-disable-next-line typescript/promise-function-async -- the walk's own promise
160export function reachableObjects(store: ObjectStore, tips: readonly string[], limits: LimitsInput = DEFAULT_LIMITS): Promise<Result<Set<string>, PlyError>> {
161 const checked = parseLimits(limits);
162 return checked.ok ? reachable({ store, limits: checked.value, source: 'the object store' }, tips) : Promise.resolve(checked);
163}
164/** Everything reachable from `tips` and not in `have`, in pack order, each tree and blob with its first path. */
165// oxlint-disable-next-line typescript/promise-function-async -- the walk's own promise
166export function objectsForPack(store: ObjectStore, tips: readonly string[], have: ReadonlySet<string> = new Set(), limits: LimitsInput = DEFAULT_LIMITS): Promise<Result<PackCandidate[], PlyError>> {
167 const checked = parseLimits(limits);
168 return checked.ok ? missing({ store, limits: checked.value, source: 'the object store' }, tips, have) : Promise.resolve(checked);
169}
Diagnostics6 git calls · 40 ms
CallStatusSizeTime040 ms
discover200191 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=11990 git.response.cache=UPDATING git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02002.4 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=29 git.response.bytes=2507 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2507 git.pkt.payload.max=2480 git.sideband.band1.bytes=2480 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=2480 git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381Open pack map
fetch2003.2 KiB · hit29 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3258 git.response.cache=HIT git.response.duration_ms=29 git.fetch.wants=e6e817c2419eeb68b8dae15e3ac085912de12c0a git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3258 git.pkt.payload.max=3231 git.sideband.band1.bytes=3231 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e14c4a1c61a140
- Request
- 392e8d39…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format