GitHubivarvong
ply
main
1// Delta compression for a pack being written: a copy/insert delta generator in the shape of git's diff-delta.c, and a
2// planner that picks each object's base the way a history push wants it. Pure and Workers-safe; nothing here reads the
3// wire. The edit path does not import it (spliceDelta is all a one-region edit needs).
4//
5// encodeDelta(base, target, maxSize) → the delta (gitformat-pack "Deltified representation"), or null when the delta
6// would be maxSize bytes or more
7// planDeltas(objects, options) → buildPack/push input: each tree and blob with a path is tried against the
8// previous object of the same type at the same path, then at the same file name
9// (a rename), then (thin, opt-in) the object the remote holds at that path
10//
11// encodeDelta indexes the base in non-overlapping 16-byte blocks (git's RABIN_WINDOW), at most 64 offsets per hash
12// bucket (HASH_LIMIT), then walks the target with a rolling hash: a block hit is verified byte by byte and extended
13// forward and back over the pending literals, and the longest match becomes a copy. What no block matches is inserted.
14// The output is exactly what applyDelta (packresolve.ts) and git's patch_delta read. Deterministic: the same inputs
15// give the same bytes.
16//
17// planDeltas is bounded: a delta chain is at most maxDepth deep (git's pack.depth default, 50; resolvePack's own bound
18// is maxDeltaChain, 64), an object over maxBytes is stored whole, and a delta is kept only when it is under half the
19// object less an object id (git's try_delta max_size), so a delta is never larger than the object it replaces. Bases
20// precede their deltas in pack order, so an in-pack base is an OFS_DELTA (base `ofs`, by index); a thin base is a
21// REF_DELTA (base `ref`, by id), and only a caller who knows the remote has that object passes one.
22/* oxlint-disable typescript/no-unsafe-type-assertion, typescript/non-nullable-type-assertion-style -- the scan's indexed reads are in range of a checked length and say `as number` (free at run time, unlike an Option per byte; not the `!` src/ forbids), as packresolve.ts does */
23import { concat } from './bytes.js';
24import { copyOps, encodeVarint, inputBase, insertOps } from './pack.js';
25import type { PackInput } from './pack.js';
26import { TYPE } from './objects.js';
27import { DELTA_INSERT_MAX, DELTA_SIZE_MIN, OID_BYTES } from './format.js';
28import type { Input } from './absent.js';
29import { at, mapGet, withDefaults } from './absent.js';
30import { getOrElse, mapOption } from './result.js';
32/** git diff-delta.c RABIN_WINDOW: the base is indexed in blocks of this many bytes, and a copy is at least this long. */
33export const DELTA_WINDOW = 16;
34/** git diff-delta.c HASH_LIMIT: offsets kept per hash bucket, so a run of identical blocks cannot make a scan quadratic. */
35export const DELTA_BUCKET_LIMIT = 64;
36/** A match this long is taken without trying the bucket's other offsets. */
37const GOOD_MATCH = 4096;
38/** Polynomial rolling hash over one window (mod 2^32). PRIME_POW is PRIME^(WINDOW-1), the weight of the byte that leaves. */
39const PRIME = 0x01_00_01_93;
40const PRIME_POW = ((): number => { let p = 1; for (let i = 1; i < DELTA_WINDOW; i++) p = Math.imul(p, PRIME); return p; })();
41const HASH_SPREAD = 15;
42const MIN_TABLE = 16;
43const NONE = -1;
45const hashAt = (buf: Uint8Array, p: number): number => {
46 let h = 0;
47 for (let k = 0; k < DELTA_WINDOW; k++) h = (Math.imul(h, PRIME) + (buf[p + k] as number)) >>> 0;
48 return h;
49};
50// Math.imul reads its operands mod 2^32, so the subtraction needs no wrap of its own
51const roll = (h: number, out: number, inp: number): number => (Math.imul(h - Math.imul(out, PRIME_POW), PRIME) + inp) >>> 0;
52const bucketOf = (h: number, mask: number): number => (h ^ (h >>> HASH_SPREAD)) & mask;
54// The base's block index: head[bucket] is the first block, next[block] the following one; -1 ends a chain.
55type Index = { readonly head: Int32Array; readonly next: Int32Array; readonly mask: number };
56function indexBase(base: Uint8Array): Index {
57 const blocks = Math.floor(base.length / DELTA_WINDOW);
58 let size = MIN_TABLE;
59 while (size < blocks) size *= 2;
60 const mask = size - 1;
61 const head = new Int32Array(size).fill(NONE);
62 const next = new Int32Array(blocks).fill(NONE);
63 const count = new Uint8Array(size);
64 // last block first, so each chain lists the lowest offsets first and the 64 kept are the earliest
65 for (let b = blocks - 1; b >= 0; b--) {
66 const k = bucketOf(hashAt(base, b * DELTA_WINDOW), mask);
67 if ((count[k] as number) >= DELTA_BUCKET_LIMIT) {
68 // a full bucket: this earlier block replaces the chain's last entry's place at the head (bounded, deterministic)
69 next[b] = next[head[k] as number] as number;
70 head[k] = b;
71 continue;
72 }
73 next[b] = head[k] as number;
74 head[k] = b;
75 count[k] = (count[k] as number) + 1;
76 }
77 return { head, next, mask };
78}
80// An insert of n bytes costs n plus one opcode per 127.
81const literalCost = (n: number): number => n + Math.ceil(n / DELTA_INSERT_MAX);
82// The longest run of target bytes from `pos` that the base holds at an offset in the hash's bucket: [offset, length].
83function longestMatch(base: Uint8Array, target: Uint8Array, pos: number, index: Index, h: number): readonly [number, number] {
84 let bestLen = 0;
85 let bestOff = 0;
86 for (let b = index.head[bucketOf(h, index.mask)] as number; b !== NONE; b = index.next[b] as number) {
87 const off = b * DELTA_WINDOW;
88 const lim = Math.min(base.length - off, target.length - pos);
89 let n = 0;
90 while (n < lim && base[off + n] === target[pos + n]) n++;
91 if (n > bestLen) { bestLen = n; bestOff = off; if (n >= GOOD_MATCH) break; }
92 }
93 return [bestOff, bestLen];
94}
96// The delta that rebuilds `target` from `base`, or null when it would be `maxSize` bytes or more (or the base is too
97// short to index). maxSize defaults to the target's length: a delta that does not save a byte is not one.
98export function encodeDelta(base: Uint8Array, target: Uint8Array, maxSize: number = target.length): Uint8Array<ArrayBuffer> | null {
99 if (base.length < DELTA_WINDOW || target.length < DELTA_WINDOW) return null;
100 const sizes = [encodeVarint(base.length), encodeVarint(target.length)];
101 const parts: Uint8Array[] = [...sizes];
102 let size = sizes.reduce((n, b) => n + b.length, 0);
103 const emit = (ops: readonly Uint8Array[]): void => { for (const o of ops) { parts.push(o); size += o.length; } };
104 const index = indexBase(base);
105 let pos = 0;
106 let lit = 0;
107 let h = hashAt(target, 0);
108 while (pos + DELTA_WINDOW <= target.length) {
109 const [bestOff, bestLen] = longestMatch(base, target, pos, index, h);
110 if (bestLen < DELTA_WINDOW) {
111 if (size + literalCost(pos + 1 - lit) >= maxSize) return null;
112 if (pos + DELTA_WINDOW < target.length) h = roll(h, target[pos] as number, target[pos + DELTA_WINDOW] as number);
113 pos++;
114 continue;
115 }
116 // extend the match back over literals not yet written
117 let back = 0;
118 while (pos - back > lit && bestOff - back > 0 && base[bestOff - back - 1] === target[pos - back - 1]) back++;
119 if (pos - back > lit) emit(insertOps(target.subarray(lit, pos - back)));
120 emit(copyOps(bestOff - back, bestLen + back));
121 if (size >= maxSize) return null;
122 pos += bestLen;
123 lit = pos;
124 if (pos + DELTA_WINDOW <= target.length) h = hashAt(target, pos);
125 }
126 if (target.length > lit) emit(insertOps(target.subarray(lit)));
127 return size < maxSize && size >= DELTA_SIZE_MIN ? concat(parts) : null;
128}
130// ---- the planner ----
131/** One object in pack order. `path` (the object's path in some commit's tree) makes a tree or blob a delta candidate. */
132export type DeltaCandidate = { readonly type: number; readonly data: Uint8Array; readonly path: string | null };
133/** A base the remote is known to hold: its id and bytes. */
134export type ThinBase = { readonly oid: string; readonly data: Uint8Array };
135export type DeltaPlanConfig = {
136 // the longest chain a delta may end (git pack.depth); 0 stores everything whole
137 readonly maxDepth: number;
138 // an object larger than this is stored whole (the generator holds an index of the base beside both)
139 readonly maxBytes: number;
140 // the remote's object for (type, path), or null. Only for a remote that accepts thin packs; null: no thin bases.
141 readonly thinBase: ((type: number, path: string) => ThinBase | null) | null;
142};
143export type DeltaPlanOptions = Input<DeltaPlanConfig>;
144export type DeltaPlan = {
145 readonly objects: PackInput[];
146 // deltas against an earlier object of this pack (OFS_DELTA) and against a thin base (REF_DELTA)
147 readonly ofs: number;
148 readonly ref: number;
149 // the bytes of every object as given, and of what the plan writes (before zlib)
150 readonly rawBytes: number;
151 readonly plannedBytes: number;
152};
153/** git pack.depth's default. */
154export const DELTA_DEPTH = 50;
155/** Objects larger than this are stored whole: the index of the base and the scan stay a bounded amount of work. */
156export const DELTA_MAX_BYTES: number = 16 * 1024 * 1024;
157/** A target smaller than this is not worth a delta's two size headers and an opcode. */
158const MIN_TARGET = 64;
160const fileName = (path: string): string => path.slice(path.lastIndexOf('/') + 1);
161const EMPTY = new Uint8Array(0);
162type Base = { readonly index: number; readonly depth: number; readonly data: Uint8Array };
163// The smallest delta of `o` against one of `bases` (in order: the first wins a tie) that keeps the chain within
164// maxDepth and is under half the object less an object id (git try_delta's max_size); null when none is.
165function bestDelta(o: DeltaCandidate, bases: readonly Base[], { maxDepth, maxBytes }: Pick<DeltaPlanConfig, 'maxDepth' | 'maxBytes'>): { readonly delta: Uint8Array; readonly index: number; readonly depth: number } | null {
166 let best: { readonly delta: Uint8Array; readonly index: number; readonly depth: number } | null = null;
167 let limit = Math.floor(o.data.length / 2) - OID_BYTES;
168 for (const b of bases) {
169 if (b.depth + 1 > maxDepth || b.data.length > maxBytes) continue;
170 const delta = encodeDelta(b.data, o.data, limit);
171 if (delta === null) continue;
172 best = { delta, index: b.index, depth: b.depth + 1 };
173 limit = delta.length;
174 }
175 return best;
176}
178// Each object's representation, in the order given. Commits and tags are stored whole, as git stores them in practice.
179export function planDeltas(objects: readonly DeltaCandidate[], options: DeltaPlanOptions = {}): DeltaPlan {
180 const { maxDepth, maxBytes, thinBase } = withDefaults<DeltaPlanConfig>(options, { maxDepth: DELTA_DEPTH, maxBytes: DELTA_MAX_BYTES, thinBase: null });
181 const out: PackInput[] = [];
182 const depth: number[] = [];
183 const lastAt = new Map<string, number>();
184 let ofs = 0;
185 let ref = 0;
186 let rawBytes = 0;
187 let plannedBytes = 0;
188 for (const [i, o] of objects.entries()) {
189 rawBytes += o.data.length;
190 let chosen: PackInput = { type: o.type, data: o.data };
191 let chosenDepth = 0;
192 const deltable = (o.type === TYPE.BLOB || o.type === TYPE.TREE) && o.data.length >= MIN_TARGET && o.data.length <= maxBytes && maxDepth > 0;
193 if (deltable && o.path !== null) {
194 const keys = [`${o.type} ${o.path}`, `${o.type} name ${fileName(o.path)}`];
195 const bases = [...new Set(keys.map(k => getOrElse(mapGet(lastAt, k), NONE)).filter(j => j !== NONE))];
196 const picked = bestDelta(o, bases.map(j => ({ index: j, depth: getOrElse(at(depth, j), 0), data: getOrElse(mapOption(at(objects, j), b => b.data), EMPTY) })), { maxDepth, maxBytes });
197 if (picked !== null) { chosen = { type: o.type, data: picked.delta, base: { tag: 'ofs', index: picked.index } }; chosenDepth = picked.depth; }
198 // the first version of a path in this pack: the remote's own copy, when the caller vouches for thin packs
199 const thin = thinBase !== null && bases.length === 0 ? thinBase(o.type, o.path) : null;
200 if (thin !== null && thin.data.length <= maxBytes) {
201 const delta = encodeDelta(thin.data, o.data, Math.floor(o.data.length / 2) - OID_BYTES);
202 if (delta !== null) { chosen = { type: o.type, data: delta, base: { tag: 'ref', oid: thin.oid } }; chosenDepth = 1; }
203 }
204 for (const key of keys) lastAt.set(key, i);
205 }
206 const tag = inputBase(chosen).tag;
207 if (tag === 'ofs') ofs++;
208 else if (tag === 'ref') ref++;
209 plannedBytes += chosen.data.length;
210 depth.push(chosenDepth);
211 out.push(chosen);
212 }
213 return { objects: out, ofs, ref, rawBytes, plannedBytes };
214}
Diagnostics8 git calls · 179 ms
CallStatusSizeTime0179 ms
discover200191 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB95 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=89 git.response.bytes=11990 git.response.cache=EXPIRED git.response.duration_ms=95 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0 · deepen 1200347 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=59 git.response.bytes=347 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=347 git.pkt.payload.max=247 git.sideband.band1.bytes=247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=247 git.pack.trailer=4b72f39b5ea57c5ae3009f90b53e3a6a155093e7Open pack map
fetchcombine:blob:none+tree:2 · deepen 1429—59 ms
git.upload-pack fetch
http.response.status_code=429
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.cache=BYPASS
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
error.type=rate_limited
error=HTTP 429 git-upload-pack: \u000d
<html>\u000d
<head>\u000d
<meta content="origin" name="referrer">\u000d
<title>Rate limit · GitHub</title>\u000d
<meta name="vfetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02002.4 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=2507 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2507 git.pkt.payload.max=2480 git.sideband.band1.bytes=2480 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=2480 git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381Open pack map
fetch2004.6 KiB · hit18 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=17 git.response.bytes=4739 git.response.cache=HIT git.response.duration_ms=18 git.fetch.wants=50c3360ee8fa443f0a741817bed390dfb1f432dd git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4739 git.pkt.payload.max=4712 git.sideband.band1.bytes=4712 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e26e499ac6f4e0
- Request
- e14129fe…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format