main
1// Pack index v2 (gitformat-pack "Version 2 pack-*.idx files"), the file `git index-pack` writes beside a pack: what a
2// dumb-HTTP client (and any git reading a pack from disk) needs to find an object without reading the pack. Pure and
3// Workers-safe; the SHA-1 is ply's own, so it is synchronous.
4//
5// encodePackIndex(pack, objects) → the .idx of a pack you just wrote (buildPack's entries give the offsets; you know
6// the ids): CRC-32s from the pack's bytes, the pack checksum from its trailer
7// indexPack(pack, limits) → the .idx of any self-contained pack (git index-pack): parsed, trailer verified,
8// every object resolved and hashed; a thin pack is pack_corrupt (no --fix-thin)
9// layoutPackIndex(rows, checksum) → the layout alone, from { oid, offset, crc32 } rows
10//
11// Byte for byte what git 2.x index-pack writes for the same pack (test/packindex.test.js), including the 64-bit offset
12// table for objects at or past 2 GiB. One object id twice is refused, as `index-pack --strict` refuses it.
13/* oxlint-disable typescript/no-unsafe-type-assertion, typescript/non-nullable-type-assertion-style -- indexed reads of a CRC table and of a sorted array within its checked length say `as number`, as packresolve.ts does */
14import { fail } from './errors.js';
15import type { PlyError } from './errors.js';
16import { DEFAULT_LIMITS, parseLimits } from './limits.js';
17import type { LimitsInput } from './limits.js';
18import { Sha1, fromHex, toHex } from './sha1.js';
19import { parsePack, resolvePack } from './packresolve.js';
20import { OID_BYTES, PACK_HEADER_BYTES, PACK_SIGNATURE, PACK_TRAILER_BYTES, PACK_VERSION } from './format.js';
21import { ok } from './result.js';
22import type { Result } from './result.js';
24/** One object of the index: its id, its entry's offset in the pack, and the CRC-32 of the entry's bytes as stored. */
25export type IndexRow = { readonly oid: string; readonly offset: number; readonly crc32: number };
26/** One object of a pack you wrote: its id and its entry's offset (buildPack's `entries[i].offset`). */
27export type IndexedObject = { readonly oid: string; readonly offset: number };
28export type IndexedPack = { readonly idx: Uint8Array<ArrayBuffer>; readonly checksum: string; readonly objects: number };
30const IDX_SIGNATURE = [0xff, 0x74, 0x4f, 0x63]; // "\377tOc"
31const IDX_VERSION = 2;
32const FANOUT = 256;
33const U32 = 4;
34const U64 = 8;
35// An offset at or past 2^31 goes to the 64-bit table; its 32-bit slot holds that table's index with the high bit set.
36const LARGE_OFFSET = 0x80_00_00_00;
37const U32_LIMIT = 0x1_00_00_00_00;
38const OID_RE = /^[0-9a-f]{40}$/u;
40// CRC-32 (ISO-HDLC, zlib's crc32: reflected, polynomial 0xEDB88320), the checksum git stores per entry.
41const CRC_POLY = 0xed_b8_83_20;
42let crcTable: Uint32Array | null = null;
43function crc32(bytes: Uint8Array): number {
44 if (crcTable === null) {
45 const t = new Uint32Array(FANOUT);
46 for (let n = 0; n < FANOUT; n++) {
47 let c = n;
48 for (let k = 0; k < 8; k++) c = c & 1 ? CRC_POLY ^ (c >>> 1) : c >>> 1;
49 t[n] = c;
50 }
51 crcTable = t;
52 }
53 const t = crcTable;
54 let c = 0xff_ff_ff_ff;
55 for (const b of bytes) c = (t[(c ^ b) & 0xff] as number) ^ (c >>> 8);
56 return (c ^ 0xff_ff_ff_ff) >>> 0;
57}
59const bad = (message: string, what: string): Result<never, PlyError> => fail('bad_request', message, { what });
61/** The .idx v2 bytes for `rows`, which name every object of the pack whose trailer is `packChecksum`. */
62export function layoutPackIndex(rows: readonly IndexRow[], packChecksum: Uint8Array): Result<Uint8Array<ArrayBuffer>, PlyError> {
63 if (packChecksum.length !== OID_BYTES) return bad(`pack index: the pack checksum is ${packChecksum.length} bytes, not ${OID_BYTES}`, 'checksum');
64 for (const r of rows) {
65 if (!OID_RE.test(r.oid)) return bad(`pack index: ${JSON.stringify(r.oid)} is not a 40-hex object id`, 'oid');
66 if (!Number.isSafeInteger(r.offset) || r.offset < PACK_HEADER_BYTES) return bad(`pack index: offset ${r.offset} of ${r.oid} is not an entry offset`, 'offset');
67 if (!Number.isInteger(r.crc32) || r.crc32 < 0 || r.crc32 >= U32_LIMIT) return bad(`pack index: crc32 ${r.crc32} of ${r.oid} is not a 32-bit value`, 'crc32');
68 }
69 const byOid = (a: IndexRow, b: IndexRow): number => {
70 if (a.oid === b.oid) return 0;
71 return a.oid < b.oid ? -1 : 1;
72 };
73 const sorted = rows.toSorted(byOid);
74 for (let i = 1; i < sorted.length; i++) {
75 const oid = (sorted[i] as IndexRow).oid;
76 if (oid === (sorted[i - 1] as IndexRow).oid) return bad(`pack index: ${oid} appears twice`, 'oid');
77 }
78 const n = sorted.length;
79 const large = sorted.filter(r => r.offset >= LARGE_OFFSET);
80 const out = new Uint8Array(IDX_SIGNATURE.length + U32 + FANOUT * U32 + n * (OID_BYTES + U32 + U32) + large.length * U64 + 2 * OID_BYTES);
81 const dv = new DataView(out.buffer);
82 out.set(IDX_SIGNATURE);
83 let p = IDX_SIGNATURE.length;
84 dv.setUint32(p, IDX_VERSION); p += U32;
85 // fanout[b]: how many ids have a first byte ≤ b
86 const counts = new Uint32Array(FANOUT);
87 for (const r of sorted) { const b = Number.parseInt(r.oid.slice(0, 2), 16); counts[b] = (counts[b] as number) + 1; }
88 let total = 0;
89 for (const c of counts) { total += c; dv.setUint32(p, total); p += U32; }
90 for (const r of sorted) { out.set(fromHex(r.oid), p); p += OID_BYTES; }
91 for (const r of sorted) { dv.setUint32(p, r.crc32); p += U32; }
92 let li = 0;
93 for (const r of sorted) {
94 dv.setUint32(p, r.offset < LARGE_OFFSET ? r.offset : (LARGE_OFFSET | li) >>> 0);
95 if (r.offset >= LARGE_OFFSET) li++;
96 p += U32;
97 }
98 for (const r of large) { dv.setBigUint64(p, BigInt(r.offset)); p += U64; }
99 out.set(packChecksum, p); p += OID_BYTES;
100 out.set(new Sha1().update(out.subarray(0, p)).digest(), p);
101 return ok(out);
102}
104// The pack's object count from its header, or a refusal when it is not a version-2 pack with room for a trailer.
105function packCount(pack: Uint8Array): Result<number, PlyError> {
106 if (pack.length < PACK_HEADER_BYTES + PACK_TRAILER_BYTES) return bad(`pack index: ${pack.length} bytes is shorter than a pack`, 'pack');
107 const dv = new DataView(pack.buffer, pack.byteOffset, pack.byteLength);
108 const signed = PACK_SIGNATURE.every((b, i) => pack[i] === b) && dv.getUint32(PACK_SIGNATURE.length) === PACK_VERSION;
109 if (!signed) return bad('pack index: not a version 2 pack', 'pack');
110 return ok(dv.getUint32(PACK_SIGNATURE.length + U32));
111}
113/**
114 * The .idx of a pack you just wrote. `objects` names every entry: its id and its offset (buildPack's entries, in pack
115 * order, carry the offsets). Each entry's bytes run to the next entry (or the trailer); their CRC-32 is computed here.
116 * Offsets are trusted as entry boundaries, not re-parsed: use indexPack for a pack from elsewhere.
117 */
118export function encodePackIndex(pack: Uint8Array, objects: readonly IndexedObject[]): Result<Uint8Array<ArrayBuffer>, PlyError> {
119 const count = packCount(pack);
120 if (!count.ok) return count;
121 if (objects.length !== count.value) return bad(`pack index: ${objects.length} objects given for a pack of ${count.value}`, 'objects');
122 const end = pack.length - PACK_TRAILER_BYTES;
123 const byOffset = objects.toSorted((a, b) => a.offset - b.offset);
124 const rows: IndexRow[] = [];
125 for (let i = 0; i < byOffset.length; i++) {
126 const o = byOffset[i] as IndexedObject;
127 const next = i + 1 < byOffset.length ? (byOffset[i + 1] as IndexedObject).offset : end;
128 if (i === 0 && o.offset !== PACK_HEADER_BYTES) return bad(`pack index: the first entry is at ${o.offset}, not ${PACK_HEADER_BYTES}`, 'offset');
129 if (!Number.isSafeInteger(o.offset) || next <= o.offset || next > end) return bad(`pack index: offset ${o.offset} of ${o.oid} is not an entry boundary`, 'offset');
130 rows.push({ oid: o.oid, offset: o.offset, crc32: crc32(pack.subarray(o.offset, next)) });
131 }
132 return layoutPackIndex(rows, pack.subarray(end));
133}
135/**
136 * The .idx of any self-contained pack, as `git index-pack` writes it: the pack is parsed under `limits` and its trailer
137 * verified, every object is resolved (a REF_DELTA base outside the pack is pack_corrupt) and hashed. One object twice is
138 * pack_corrupt.
139 */
140export function indexPack(pack: Uint8Array, limits: LimitsInput = DEFAULT_LIMITS): Result<IndexedPack, PlyError> {
141 const checked = parseLimits(limits);
142 if (!checked.ok) return checked;
143 const parsed = parsePack(pack, { limits: checked.value });
144 if (!parsed.ok) return parsed;
145 const resolved = resolvePack(parsed.value, () => null, checked.value);
146 if (!resolved.ok) return resolved;
147 if (resolved.value.objects.size !== parsed.value.objects.length) return fail('pack_corrupt', `pack index: ${parsed.value.objects.length - resolved.value.objects.size} objects appear twice in the pack`, { what: 'object' });
148 const objects = [...resolved.value.objects.values()].map(o => ({ oid: o.oid, offset: o.offset }));
149 const idx = encodePackIndex(pack, objects);
150 if (!idx.ok) return idx;
151 return ok({ idx: idx.value, checksum: toHex(pack.subarray(pack.length - PACK_TRAILER_BYTES)), objects: objects.length });
152}
Diagnostics4 git calls · 510 ms
CallStatusSizeTime0510 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=505 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2003.3 KiB505 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3373 git.response.cache=MISS git.response.duration_ms=505 git.fetch.wants=d5fc22f9148801ea31a1764b66a14cf2e0ce5a7d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3373 git.pkt.payload.max=3352 git.sideband.band1.bytes=3351 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2de808f9b2308
- Request
- e900f50d…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format