Artifactspublic-v0

ply

main
ply4 git calls541 ms43 KiB2 cachedprotocol v2
307 lines16 KiBRaw
1// parsePack as ply had it before #172 made it a collector over the pack scan (src/packstream.ts): a whole buffer walked
2// with the old inflate (inflate-ref.ts), kept in test/ as an independent oracle for the scan. One change from the old
3// code, the one #156 made on purpose: the declared size is refused right after the object header (before the base),
4// where readObjHeader now checks it. Its objects carry baseOffset/baseOid; test/packstream.test.js reads them as the
5// tagged base.
6import type { ErrorOf } from '../../src/errors.js';
7import { fail } from '../../src/errors.js';
8import type { Err, Result } from '../../src/result.js';
9import { ok, err, assertNever, getOrElse } from '../../src/result.js';
10import { at, atOr, fromNullable, fromNullish, readKey, type Input } from '../../src/absent.js';
11import type { InflateStats } from '../../src/inflatestream.js';
12import { inflateRef } from './inflate-ref.js';
13import { Sha1, toHex, toOid, type Oid } from '../../src/sha1.js';
14import { nameOfTypeId } from '../../src/objects.js';
15import type { ObjType } from '../../src/objects.js';
16import type { LimitKey, Limits } from '../../src/limits.js';
17import { DEFAULT_LIMITS, LIMIT_OBJECT_OVERHEAD, limitError, checkLimit } from '../../src/limits.js';
19// One key of a caller's limits record as src/ reads it since #156 (parseLimits): a key given must pass checkLimit, a key
20// left out is DEFAULT_LIMITS's. The oracle reads one key at a time, where src/ parses the record at the edge.
21const readLimit = (limits: Input<Limits> | null, key: LimitKey): Result<number, ErrorOf<'bad_request'>> =>
22 checkLimit(getOrElse<unknown>(fromNullable((limits === null ? {} : limits)[key]), DEFAULT_LIMITS[key]), key);
23import { take, shortRead } from '../../src/bytes.js';
24import type { PackKind } from '../../src/format.js';
25import {
26 PACK_SIGNATURE, PACK_VERSION, PACK_HEADER_BYTES, PACK_TRAILER_BYTES, OID_BYTES, UINT32_BYTES,
27 OBJ_TYPE_SHIFT, OBJ_TYPE_MASK, OBJ_SIZE_FIRST_MASK, OBJ_SIZE_NEXT_MASK, OBJ_CONTINUE,
28 OBJ_SIZE_NEXT_BASE, VARINT_BITS, VARINT_MAX_SHIFT, varintScale, BE32_SHIFTS,
29} from '../../src/format.js';
30import { TYPE } from '../../src/objects.js';
32type Corrupt = ErrorOf<'pack_corrupt'>;
33// Everything a buffered pack can be refused for: corrupt bytes, a limit, or a caller's limit that bounds nothing.
34export type PackError = Corrupt | ErrorOf<'limit_exceeded'> | ErrorOf<'bad_request'>;
35const corrupt = (msg: string): Err<Corrupt> => fail('pack_corrupt', msg);
36const short = (buf: Uint8Array, p: number): Err<Corrupt> => err(shortRead('pack_corrupt', p, 1, buf.length));
37// Indexed reads in range of a checked length say `as number` (free at runtime, unlike a fallback per byte; not the `!`
38// src/ forbids). A read that may run past the end reads atOr(buf, p, NO_BYTE): NO_BYTE there is a short read.
39const NO_BYTE = -1;
41// The six pack object types git writes (pack.h OBJ_*): four objects and two deltas. 0 and 5 name nothing.
42type HeaderType = PackKind | 'invalid';
43const headerType = (n: number): HeaderType => {
44 switch (n) {
45 case TYPE.COMMIT:
46 case TYPE.TREE:
47 case TYPE.BLOB:
48 case TYPE.TAG:
49 case TYPE.OFS_DELTA:
50 case TYPE.REF_DELTA:
51 return nameOfTypeId(n);
52 default:
53 return 'invalid';
54 }
55};
56// A type id this pack will hash. An id outside the six is corrupt, the same refusal as a bad header.
57const objectName = (type: number): Result<PackKind, Corrupt> => {
58 const kind = headerType(type);
59 switch (kind) {
60 case 'invalid': return corrupt(`pack: bad object type ${type}`);
61 case 'commit':
62 case 'tree':
63 case 'blob':
64 case 'tag':
65 case 'ofs_delta':
66 case 'ref_delta':
67 return ok(kind);
68 }
69 return assertNever(kind);
70};
72// Past VARINT_MAX_SHIFT a group used to scale by Infinity, and 0 × Infinity is NaN: every `size > limit` check is false
73// on NaN, so the size reached callers (#13). Checked before the byte is read, so a hostile header costs ≤ 10 bytes.
74const varintLong = (what: string): Err<Corrupt> => corrupt(`${what}: size varint longer than ${VARINT_MAX_SHIFT + 1} bits`);
76// Little-endian 7-bit groups from start, each with a continue bit: the delta header's sizes (from n = 0, shift = 0), and
77// the rest of an object header's size after a first byte that continues (from that byte's four bits, shift = 4).
78function readVarint(buf: Uint8Array, start: number, what: string, n0 = 0, shift0 = 0): Result<readonly [number, number], Corrupt> {
79 let p = start;
80 let n = n0;
81 let shift = shift0;
82 let c: number;
83 do {
84 if (shift > VARINT_MAX_SHIFT) return varintLong(what);
85 c = atOr(buf, p, NO_BYTE);
86 if (c === NO_BYTE) return short(buf, p);
87 p += 1;
88 n += (c & OBJ_SIZE_NEXT_MASK) * varintScale(shift);
89 shift += VARINT_BITS;
90 } while (c & OBJ_CONTINUE);
91 return ok([n, p]);
94// Four bytes at off, big-endian, where the caller has checked they are there.
95const be32 = (b: Uint8Array, off: number): number =>
96 // b holds four bytes at off and BE32_SHIFTS (format.js) has four entries: in range, see above
97 /* oxlint-disable-next-line typescript/no-unsafe-type-assertion, typescript/non-nullable-type-assertion-style, no-magic-numbers -- see above; off + 1 and off + 2 are the middle bytes */
98 (((b[off] as number) << (BE32_SHIFTS[0] as number)) | ((b[off + 1] as number) << (BE32_SHIFTS[1] as number)) | ((b[off + 2] as number) << (BE32_SHIFTS[2] as number)) | (b[off + BE32_SHIFTS.length - 1] as number)) >>> 0;
99export function readUint32BE(buf: Uint8Array, off: number): Result<number, Corrupt> {
100 const r = take(buf, off, BE32_SHIFTS.length);
101 return r.ok ? ok(be32(buf, off)) : r;
104// The header's three type bits as one of the six pack types, or null for 0 and 5 (which name nothing).
105function objTypeOf(n: number): ObjType | null {
106 switch (n) {
107 case TYPE.COMMIT:
108 case TYPE.TREE:
109 case TYPE.BLOB:
110 case TYPE.TAG:
111 case TYPE.OFS_DELTA:
112 case TYPE.REF_DELTA:
113 return n;
114 default:
115 return null;
116 }
118// `type` is parsed here, once: a reader of an ObjHeader sees one of the six pack types and no other number.
119export type ObjHeader = { readonly type: ObjType; readonly size: number; readonly next: number };
120// A header cut by the end of the buffer is a short read (`reason: 'short_read'`): a stream parser waits for more bytes
121// on that reason (fetch.ts spStep), and refuses anything else. A whole header whose type is 0 or 5 is
122// `pack: bad object type <n> at <offset>`, after the size (a size varint past 64 bits is refused first, as it always was);
123// `offset` is the header's offset in the pack, which a stream parser's carry buffer does not start at.
124export function readObjHeader(buf: Uint8Array, start: number, offset: number = start): Result<ObjHeader, Corrupt> {
125 const c = atOr(buf, start, NO_BYTE);
126 if (c === NO_BYTE) return short(buf, start);
127 const bits = (c >> OBJ_TYPE_SHIFT) & OBJ_TYPE_MASK;
128 let size = c & OBJ_SIZE_FIRST_MASK;
129 let next = start + 1;
130 if (c & OBJ_CONTINUE) {
131 const rest = readVarint(buf, start + 1, 'pack', size, OBJ_TYPE_SHIFT);
132 if (!rest.ok) return rest;
133 [size, next] = rest.value;
134 }
135 const type = objTypeOf(bits);
136 if (type === null) return corrupt(`pack: bad object type ${bits} at ${offset}`);
137 return ok({ type, size, next });
140export type OfsDelta = { readonly off: number; readonly next: number };
141// The same ten-byte cap as the size varints (readObjHeader): git's index-pack refuses an offset that overflows
142// ("offset value overflow for delta base object"); past it the offset could only name a base before the pack start.
143export function readOfsDelta(buf: Uint8Array, start: number): Result<OfsDelta, Corrupt> {
144 let p = start;
145 let c = atOr(buf, p, NO_BYTE);
146 if (c === NO_BYTE) return short(buf, p);
147 p += 1;
148 let off = c & OBJ_SIZE_NEXT_MASK;
149 let shift = VARINT_BITS;
150 while (c & OBJ_CONTINUE) {
151 if (shift > VARINT_MAX_SHIFT) return corrupt(`pack: ofs delta offset longer than ${VARINT_MAX_SHIFT + 1} bits`);
152 c = atOr(buf, p, NO_BYTE);
153 if (c === NO_BYTE) return short(buf, p);
154 p += 1;
155 off = ((off + 1) * OBJ_SIZE_NEXT_BASE) + (c & OBJ_SIZE_NEXT_MASK);
156 shift += VARINT_BITS;
157 }
158 return ok({ off, next: p });
161// ---- parse ----
162// Caps are required. A missing field used to become Infinity (a nullish fallback), which disabled the check; checkLimit rejects that.
163// Each key's cap is the caller's override at the same index when it gave one, else limits[key]; the first cap that
164// bounds nothing is the error. One cap per key, in order, so the result has keys.length entries.
165type Caps = readonly [number, number, number];
166function capsOf(limits: Limits, keys: readonly [LimitKey, LimitKey, LimitKey], given: readonly unknown[] = []): Result<Caps, ErrorOf<'bad_request'>> {
167 const out: number[] = [];
168 for (const [i, k] of keys.entries()) {
169 const g = at(given, i);
170 const r = g.tag === 'some' ? checkLimit(g.value, k) : readLimit(limits, k);
171 if (!r.ok) return r;
172 out.push(r.value);
173 }
174 // three keys, three caps (the loop above)
175 // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- see above
176 return ok(out as unknown as Caps);
178// verify: check the trailer's SHA-1 (anything but false verifies); limits: the caps (DEFAULT_LIMITS); maxObjects and
179// maxInflatedTotal: a cap that overrides the one in limits.
180export type PackParseConfig = {
181 readonly verify: boolean;
182 readonly limits: Limits;
183 readonly maxObjects: number;
184 readonly maxInflatedTotal: number;
185};
186export type PackParseOptions = Input<PackParseConfig>;
187// parsePack's options for `limits`: the two pack caps read out of it, checked (the first that bounds nothing is the error).
188export type PackParseOpts = { readonly limits: Limits; readonly maxObjects: number; readonly maxInflatedTotal: number };
189export function packParseOpts(limits: Limits = DEFAULT_LIMITS): Result<PackParseOpts, ErrorOf<'bad_request'>> {
190 const maxObjects = readLimit(limits, 'maxPackObjects');
191 if (!maxObjects.ok) return maxObjects;
192 const maxInflatedTotal = readLimit(limits, 'maxInflatedTotal');
193 if (!maxInflatedTotal.ok) return maxInflatedTotal;
194 return ok({ limits, maxObjects: maxObjects.value, maxInflatedTotal: maxInflatedTotal.value });
197// One object as parsePack read it. A full object has neither base (both null); an OFS_DELTA has baseOffset, a REF_DELTA
198// baseOid. A base is read by its type (hasOfsBase, hasRefBase), so a hand-built object that leaves a base out is a full one.
199export type PackObject = {
200 readonly offset: number;
201 readonly headerBytes: number;
202 readonly type: number;
203 readonly size: number;
204 readonly compressed: number;
205 readonly data: Uint8Array;
206 readonly baseOffset: number | null;
207 readonly baseOid: string | null;
208 readonly inflate: InflateStats;
209};
210type Ofs = { readonly baseOffset: number };
211type Ref = { readonly baseOid: string };
212const hasOfsBase = <T extends { readonly baseOffset: number | null }>(o: T): o is T & Ofs => typeof o.baseOffset === 'number';
213const hasRefBase = <T extends { readonly baseOid: string | null }>(o: T): o is T & Ref => typeof o.baseOid === 'string';
214export type PackStats = {
215 readonly version: number;
216 readonly count: number;
217 full: number;
218 ofsDelta: number;
219 refDelta: number;
220 readonly byType: Record<string, number>;
221 compressed: number;
222 inflated: number;
223 readonly inflate: { blocks: number; stored: number; fixed: number; dynamic: number };
224};
225// What parsePack read: the type is one of the six and a REF base is an Oid (parsed where the bytes were read). A
226// hand-built PackObject (resolvePack's input) is still checked by the resolver.
227export type ParsedObject = PackObject & { readonly type: ObjType; readonly baseOid: Oid | null };
228export type ParsedPack = { readonly objects: ParsedObject[]; readonly stats: PackStats; readonly trailer: string; readonly bytes: number };
230export function parsePackRef(buf: Uint8Array, opts: PackParseOptions = {}): Result<ParsedPack, PackError> {
231 const verify = opts.verify !== false;
232 const limits = getOrElse(fromNullish(opts.limits), DEFAULT_LIMITS);
233 const caps = capsOf(limits, ['maxPackObjects', 'maxInflatedTotal', 'maxObjectSize'], [opts.maxObjects, opts.maxInflatedTotal]);
234 if (!caps.ok) return caps;
235 const [maxObjects, maxInflatedTotal, maxObjectSize] = caps.value;
236 const minPack = PACK_HEADER_BYTES + PACK_TRAILER_BYTES;
237 // Past the minPack check the header is a view, not a bounded read; the object loop's reads are bounded.
238 if (buf.length < minPack || !buf.subarray(0, PACK_SIGNATURE.length).every((b, i) => b === PACK_SIGNATURE[i])) return corrupt('pack: bad signature');
239 // both reads are inside minPack bytes (above): views, not bounded reads
240 const version = be32(buf, PACK_SIGNATURE.length);
241 const count = be32(buf, PACK_SIGNATURE.length + UINT32_BYTES);
242 if (version !== PACK_VERSION) return corrupt(`pack: unsupported version ${version}`);
243 // A hostile count must fail before the loop allocates one object record.
244 if (count > maxObjects) return err(limitError('pack objects', count, maxObjects));
245 const objects: ParsedObject[] = [];
246 // What the objects held so far cost: declared bytes plus LIMIT_OBJECT_OVERHEAD each (the record, its views, its
247 // resolved form), so a pack of many tiny objects meets maxInflatedTotal too (#142). stats.inflated stays git's sum.
248 let charged = 0;
249 const stats: PackStats = { version, count, full: 0, ofsDelta: 0, refDelta: 0, byType: {}, compressed: 0, inflated: 0, inflate: { blocks: 0, stored: 0, fixed: 0, dynamic: 0 } };
250 let p = PACK_HEADER_BYTES;
251 for (let i = 0; i < count; i++) {
252 const offset = p;
253 const header = readObjHeader(buf, p);
254 if (!header.ok) return header;
255 const { type, size } = header.value;
256 p = header.value.next;
257 if (size > maxObjectSize) return err(limitError('pack object size', size, maxObjectSize));
258 let baseOffset: number | null = null;
259 let baseOid: Oid | null = null;
260 const kindResult = ((): Result<PackKind, PackError> => {
261 const raw = nameOfTypeId(type);
262 switch (raw) {
263 case 'commit': case 'tree': case 'blob': case 'tag': stats.full++; return ok(raw);
264 case 'ofs_delta': {
265 const ofs = readOfsDelta(buf, p);
266 if (!ofs.ok) return ofs;
267 p = ofs.value.next;
268 baseOffset = offset - ofs.value.off;
269 // git index-pack: "delta base offset is out of bound" unless 0 < base < this object. An offset of 0 names the
270 // object itself, which used to recurse to the chain limit and come back limit_exceeded instead of corrupt.
271 if (ofs.value.off === 0) return corrupt(`pack: ofs delta at ${offset} is its own base`);
272 if (baseOffset < PACK_HEADER_BYTES) return corrupt(`pack: ofs delta base before pack start at ${offset}`);
273 stats.ofsDelta++;
274 return ok(raw);
275 }
276 case 'ref_delta': {
277 // a view: the pack's 20-byte trailer is still ahead of p (buf holds minPack bytes, and every object header byte
278 // read so far came before the trailer), so OID_BYTES at p are inside buf; the trailer check below places them
279 baseOid = toOid(buf.subarray(p, p + OID_BYTES)); p += OID_BYTES;
280 stats.refDelta++;
281 return ok('ref_delta');
282 }
283 }
284 return assertNever(raw);
285 })();
286 if (!kindResult.ok) return kindResult;
287 const kind = kindResult.value;
288 // Declared inflated size is known before zlib runs, so an inflate bomb is refused before the output buffer exists.
289 const cost = size + LIMIT_OBJECT_OVERHEAD;
290 if (charged + cost > maxInflatedTotal) return err(limitError('pack inflated bytes', charged + cost, maxInflatedTotal));
291 const z = inflateRef(buf, p, { expectedSize: size, maxOutput: maxObjectSize });
292 if (!z.ok) return z;
293 const { out, consumed, stats: is } = z.value;
294 stats.compressed += consumed; stats.inflated += size; charged += cost;
295 stats.inflate.blocks += is.blocks; stats.inflate.stored += is.stored; stats.inflate.fixed += is.fixed; stats.inflate.dynamic += is.dynamic;
296 stats.byType[kind] = getOrElse(readKey(stats.byType, kind), 0) + 1;
297 objects.push({ offset, headerBytes: p - offset, type, size, compressed: consumed, data: out, baseOffset, baseOid, inflate: is });
298 p += consumed;
299 }
300 if (p + PACK_TRAILER_BYTES !== buf.length) return corrupt(`pack: expected trailer at ${p}, pack length ${buf.length}`);
301 // the check above fixed the trailer's place: these are views
302 const trailer = toHex(buf.subarray(p));
303 const computed = verify ? new Sha1().update(buf.subarray(0, p)).hex() : null;
304 if (verify && computed !== trailer) return corrupt(`pack: trailer sha mismatch ${trailer} != ${computed}`);
305 return ok({ objects, stats, trailer, bytes: buf.length });
Diagnostics4 git calls · 541 ms
CallStatusSizeTime0541 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=38434
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=38434
git.pkt.payload.max=38339
git.sideband.band1.bytes=38338
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=63
git.pack.bytes=38338
git.pack.ofs_delta=0
git.pack.ref_delta=8
git.pack.chain.max=8
git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77
git.pack.full=55
Open pack map
fetch2005.6 KiB536 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=536
git.response.bytes=5718
git.response.cache=MISS
git.response.duration_ms=536
git.fetch.wants=524121234d45cc98b671b1ad508fa8d35b8821dd
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5718
git.pkt.payload.max=5697
git.sideband.band1.bytes=5696
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2d1c7c988cf3a
Request
a2c062a9…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format