GitHubivarvong

ply

main
ply6 git calls93 ms23 KiB3 cachedprotocol v2
119 lines7.8 KiBRaw
1// Packfile v2: parse, delta resolve/apply/generate, build. Pure except buildPack (async zlib).
2import { Sha1, fromHex } from './sha1.js';
3import { TYPE, TYPE_NAME } from './objects.js';
4import { bufferSource, concat } from './bytes.js';
5import { at, fromNullish, readKey, type Input } from './absent.js';
6import { assertNever, getOrElse } from './result.js';
7import {
8 PACK_SIGNATURE, PACK_VERSION, PACK_HEADER_BYTES, OBJ_TYPE_SHIFT, OBJ_SIZE_FIRST_MASK,
9 OBJ_SIZE_NEXT_MASK, OBJ_CONTINUE, OBJ_SIZE_FIRST_BASE, OBJ_SIZE_NEXT_BASE, BE32_SHIFTS,
10} from './format.js';
11// ---- varints ----
12export function encodeObjHeader(type: number, size: number): Uint8Array {
13 let rest = size;
14 let last = (type << OBJ_TYPE_SHIFT) | (rest & OBJ_SIZE_FIRST_MASK);
15 const out: number[] = [];
16 rest = Math.floor(rest / OBJ_SIZE_FIRST_BASE);
17 // `last` is out's last element, held apart until it is final (its continue bit depends on what follows)
18 while (rest > 0) { out.push(last | OBJ_CONTINUE); last = rest & OBJ_SIZE_NEXT_MASK; rest = Math.floor(rest / OBJ_SIZE_NEXT_BASE); }
19 out.push(last);
20 return Uint8Array.from(out);
22// The delta-instruction encoders live in a leaf module (deltaenc.ts), which the resolver imports too; re-exported here,
23// where callers have always found them.
24export { encodeVarint, copyOp, copyOps, insertOps, encodeDeltaOps, spliceDelta } from './deltaenc.js';
25export type { DeltaOp, Splice, SplicedDelta } from './deltaenc.js';
27// ---- build ----
28// Any Uint8Array in; a Blob takes only an ArrayBuffer-backed view, which bufferSource guarantees (bytes.ts).
29export async function zlibDeflate(bytes: Uint8Array): Promise<Uint8Array<ArrayBuffer>> {
30 const ab = await new Response(new Blob([bufferSource(bytes)]).stream().pipeThrough(new CompressionStream('deflate'))).arrayBuffer();
31 return new Uint8Array(ab);
34// One object for buildPack: its type (1-4) and data, and its base, a tagged union with the read side's tags (ObjBase,
35// packstream.ts). none (or left out): the object whole. ofs: an OFS_DELTA whose `data` is the delta against an earlier
36// object of the same list, named by its index (its offset is not known until the pack is written). ref: a REF_DELTA
37// whose `data` is the delta against the object `oid` (a thin base the receiver holds). buildPack is the encoder and
38// checks neither: push's planPush refuses an ofs base that names no earlier object and a ref base that is not an object
39// id before anything is built.
40export type PackInputBase = { readonly tag: 'none' } | { readonly tag: 'ofs'; readonly index: number } | { readonly tag: 'ref'; readonly oid: string };
41export type PackInput = { readonly type: number; readonly data: Uint8Array } & Input<{ readonly base: PackInputBase }>;
42export const WHOLE: PackInputBase = Object.freeze({ tag: 'none' });
43// The base an object gives (left out: none).
44export const inputBase = (o: PackInput): PackInputBase => getOrElse<PackInputBase>(fromNullish(o.base), WHOLE);
45// typeName: the type's name for the five types a pack holds (TYPE_NAME), null for any other id a caller passed.
46// offset: where the entry starts in the pack (its header), as an index names it (packindex.ts encodePackIndex).
47export type BuiltEntry = {
48 readonly type: number; readonly typeName: string | null; readonly size: number; readonly compressed: number;
49 readonly baseOid: string | null; readonly bytes: number; readonly offset: number;
50};
51export const typeNameOf = (type: number): string | null => {
52 const named = readKey(TYPE_NAME as Readonly<Record<number, string>>, type);
53 return named.tag === 'some' ? named.value : null;
54};
55export type BuiltPack = { readonly pack: Uint8Array; readonly entries: BuiltEntry[] };
56// An OFS_DELTA's distance back to its base (gitformat-pack "offset encoding"): big-endian 7-bit groups, each group but
57// the last with the high bit set, and every group after the first offset by one (pack-objects write_no_reuse_object).
58export function encodeOfsDistance(distance: number): Uint8Array {
59 const out: number[] = [distance & OBJ_SIZE_NEXT_MASK];
60 let rest = Math.floor(distance / OBJ_SIZE_NEXT_BASE);
61 while (rest > 0) { rest -= 1; out.unshift(OBJ_CONTINUE | (rest & OBJ_SIZE_NEXT_MASK)); rest = Math.floor(rest / OBJ_SIZE_NEXT_BASE); }
62 return Uint8Array.from(out);
64// The 12-byte pack header for `count` objects.
65export function packHeader(count: number): Uint8Array<ArrayBuffer> {
66 const hdr = new Uint8Array(PACK_HEADER_BYTES);
67 hdr.set(PACK_SIGNATURE);
68 const dv = new DataView(hdr.buffer);
69 dv.setUint32(PACK_SIGNATURE.length, PACK_VERSION);
70 dv.setUint32(PACK_SIGNATURE.length + BE32_SHIFTS.length, count);
71 return hdr;
73// One entry before its zlib stream, the one encoder buildPack and the streamed writer (packwrite.ts) share: its pack
74// type, its object header and base bytes (`head`), and the REF base it names (BuiltEntry.baseOid). `offset` is where the
75// entry starts; `offsets` the earlier entries' (an OFS base's distance back; an index with no entry is a distance of 0).
76export type EntryHead = { readonly type: number; readonly head: Uint8Array; readonly baseOid: string | null };
77export function entryHead(o: PackInput, offset: number, offsets: readonly number[]): EntryHead {
78 const base = inputBase(o);
79 switch (base.tag) {
80 case 'none': return { type: o.type, head: encodeObjHeader(o.type, o.data.length), baseOid: null };
81 case 'ofs': return { type: TYPE.OFS_DELTA, head: concat([encodeObjHeader(TYPE.OFS_DELTA, o.data.length), encodeOfsDistance(offset - getOrElse(at(offsets, base.index), offset))]), baseOid: null };
82 case 'ref': return { type: TYPE.REF_DELTA, head: concat([encodeObjHeader(TYPE.REF_DELTA, o.data.length), fromHex(base.oid)]), baseOid: base.oid };
83 }
84 // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check
85 return assertNever(base);
87// The entry the header and its zlib stream make: what buildPack and the streamed writer report for it.
88export const builtEntry = (h: EntryHead, size: number, compressed: number, offset: number): BuiltEntry =>
89 ({ type: h.type, typeName: typeNameOf(h.type), size, compressed, baseOid: h.baseOid, bytes: h.head.length + compressed, offset });
90// Measured (test/pure.test.js history): hand-rolled stored-block zlib never beats CompressionStream's output — zlib
91// already emits stored blocks for incompressible input and ties at ≥32 bytes, and loses by 3 bytes below. So: deflate only.
92export async function buildPack(objects: readonly PackInput[]): Promise<BuiltPack> {
93 const parts: Uint8Array[] = [packHeader(objects.length)];
94 const entries: BuiltEntry[] = [];
95 // each object's offset in the pack, for an OFS_DELTA's distance back
96 const offsets: number[] = [];
97 let offset = PACK_HEADER_BYTES;
98 for (const o of objects) {
99 const h = entryHead(o, offset, offsets);
100 // one at a time, in pack order: the pack is a sequence (parallel deflate is a perf change, not this one)
101 // oxlint-disable-next-line no-await-in-loop -- see above
102 const z = await zlibDeflate(o.data);
103 parts.push(h.head, z);
104 const entry = builtEntry(h, o.data.length, z.length, offset);
105 entries.push(entry);
106 offsets.push(offset);
107 offset += entry.bytes;
108 }
109 const body = concat(parts);
110 const pack = concat([body, new Sha1().update(body).digest()]);
111 return { pack, entries };
115// General-purpose parsing/resolution, kept out of this module so bundles of the edit path do not carry it.
116// The resolver is re-exported for callers; it imports nothing from this module (the delta encoders are deltaenc.ts).
117// oxlint-disable-next-line import/no-cycle -- see above
118export { parsePack, applyDelta, parseDelta, applyProgram, deltaOps, makeDelta, resolvePack, describePack, readObjHeader, readOfsDelta, readUint32BE, packParseOpts } from './packresolve.js';
119export type { PackLayout, PackLayoutObject, PackBase, ResolvedObjType, ObjBase, DeltaProgram, PackObjectInput, ParsedObject } from './packresolve.js';
Diagnostics6 git calls · 93 ms
CallStatusSizeTime093 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02002.4 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=84
git.response.bytes=2507
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2507
git.pkt.payload.max=2480
git.sideband.band1.bytes=2480
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=2480
git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381
Open pack map
fetch2003.1 KiB84 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3183
git.response.cache=MISS
git.response.duration_ms=84
git.fetch.wants=b28c012dccebc34c53a7dd60020f39b7d5707fc3
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3183
git.pkt.payload.max=3156
git.sideband.band1.bytes=3156
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0fb416fcd332c
Request
671f4738…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format