GitHubivarvong

ply

main
ply6 git calls79 ms24 KiB3 cachedprotocol v2
186 lines10 KiBRaw
1// Streaming SHA-1. Needed because crypto.subtle.digest is whole-buffer only,
2// and pack trailers / object ids must be computed over data we never hold at once.
4import { HEXV, td, te } from './bytes.js';
5import { atOr, field } from './absent.js';
6import { flatMapOption } from './result.js';
7import { digestOid } from './oid.js';
8import type { Oid } from './oid.js';
9// toOid's and hex()'s result, for a reader of digests that needs no other part of oid.ts
10export type { Oid };
12// Workers' crypto.DigestStream: a WritableStream whose `digest` is a promise of the ArrayBuffer. Not in lib.dom, and
13// not on node or in browsers, so it is read as a field of the global (None there), never declared on Crypto.
14type DigestStream = new (algorithm: string) => WritableStream<Uint8Array> & { readonly digest: Promise<ArrayBuffer> };
15const isDigestStream = (ctor: unknown): ctor is DigestStream => typeof ctor === 'function';
17// The digits go into one buffer, decoded once: a flat string. Appending 20 two-character pieces built a rope (a V8
18// ConsString, ~15 cells): ~480 B per oid instead of ~57, and a listTree of 200 000 entries keeps one per entry, which
19// was 110 MiB retained and an OOM at a 128 MiB heap (#140).
20const DIGITS = te.encode('0123456789abcdef');
21export function toHex(bytes: Uint8Array): string {
22 const out = new Uint8Array(bytes.length * 2);
23 for (let i = 0; i < bytes.length; i++) {
24 // oxlint-disable-next-line typescript/no-unsafe-type-assertion, typescript/non-nullable-type-assertion-style -- i < bytes.length, and a nibble indexes DIGITS (16 entries); `as`, not the `!` src/ forbids
25 const b = bytes[i] as number, hi = DIGITS[b >> 4] as number, lo = DIGITS[b & 15] as number;
26 out[2 * i] = hi;
27 out[2 * i + 1] = lo;
28 }
29 return td.decode(out);
31// OID_BYTES bytes in hand (a tree entry's id, a REF_DELTA base) as the Oid they spell: two lowercase digits per byte.
32// The caller holds exactly OID_BYTES (a take or a view the frame check placed).
33export const toOid = (bytes: Uint8Array): Oid => digestOid(toHex(bytes));
34// Table-driven (bytes.ts HEXV): 4.8× faster than parseInt per byte (PR body). A pair that is not two hex digits keeps
35// parseInt's reading, so any string maps to the bytes it always did; fromHex has never validated (callers check oids).
36export function fromHex(hex: string): Uint8Array {
37 const out = new Uint8Array(hex.length >> 1);
38 for (let i = 0; i < out.length; i++) {
39 // oxlint-disable-next-line unicorn/prefer-code-point -- a code unit indexes the byte table; one past it reads -1
40 const hi = atOr(HEXV, hex.charCodeAt(2 * i), -1);
41 // oxlint-disable-next-line unicorn/prefer-code-point -- see above
42 const lo = atOr(HEXV, hex.charCodeAt(2 * i + 1), -1);
43 out[i] = (hi | lo) < 0 ? Number.parseInt(hex.slice(2 * i, 2 * i + 2), 16) : (hi << 4) | lo;
44 }
45 return out;
48// The state is five int32 fields and the schedule a 16-word Int32Array ring (#214): a Uint32Array read of a word at or
49// above 2^31 is not a small integer to V8, so every round boxed a number (the profile's SHA-1 heap share), and an
50// 80-word schedule was 64 more stores and loads per block. Same digest (test/sha1.test.js holds it to node:crypto).
51export class Sha1 {
52 // H0..H4 as int32: 0xefcdab89, 0x98badcfe and 0xc3d2e1f0 less 2^32
53 private h0 = 0x67_45_23_01;
54 private h1 = -0x10_32_54_77;
55 private h2 = -0x67_45_23_02;
56 private h3 = 0x10_32_54_76;
57 private h4 = -0x3c_2d_1e_10;
58 private readonly buf = new Uint8Array(64);
59 private bufLen = 0;
60 private len = 0;
61 private readonly w = new Int32Array(16);
63 // Bytes only: a string has no bytes until a caller encodes it (bytes.ts encodeBytes, a Result).
64 public update(bytes: Uint8Array): this {
65 this.len += bytes.length;
66 let i = 0;
67 if (this.bufLen) {
68 const n = Math.min(64 - this.bufLen, bytes.length);
69 this.buf.set(bytes.subarray(0, n), this.bufLen);
70 this.bufLen += n;
71 i = n;
72 if (this.bufLen < 64) return this;
73 this.block(this.buf, 0);
74 this.bufLen = 0;
75 }
76 for (; i + 64 <= bytes.length; i += 64) this.block(bytes, i);
77 if (i < bytes.length) {
78 this.buf.set(bytes.subarray(i));
79 this.bufLen = bytes.length - i;
80 }
81 return this;
82 }
84 // Every index below is in range (p holds 64 bytes at o, w is 16 words and every index is masked to 15), which
85 // noUncheckedIndexedAccess cannot see; `as number` says so at no runtime cost (a fallback per read is bundle bytes and
86 // a branch in the hot loop). `| 0` is the 32-bit wrap SHA-1 adds modulo, not a truncation.
87 /* oxlint-disable typescript/no-unsafe-type-assertion, typescript/non-nullable-type-assertion-style, unicorn/prefer-math-trunc -- in-range indexed reads and the 32-bit wrap, see above (`as number`, not the `!` src/ forbids) */
88 private block(p: Uint8Array, o: number): void {
89 const w = this.w;
90 for (let i = 0; i < 16; i++) {
91 w[i] = ((p[o + 4 * i] as number) << 24) | ((p[o + 4 * i + 1] as number) << 16) | ((p[o + 4 * i + 2] as number) << 8) | (p[o + 4 * i + 3] as number);
92 }
93 let a = this.h0;
94 let b = this.h1;
95 let c = this.h2;
96 let d = this.h3;
97 let e = this.h4;
98 // One loop with a branch per round; the schedule word for round i >= 16 is computed in place in the ring (w[i & 15]
99 // held w[i - 16])
100 for (let i = 0; i < 80; i++) {
101 let x: number;
102 if (i < 16) x = w[i] as number;
103 else {
104 const y = (w[(i - 3) & 15] as number) ^ (w[(i - 8) & 15] as number) ^ (w[(i - 14) & 15] as number) ^ (w[i & 15] as number);
105 x = (y << 1) | (y >>> 31);
106 w[i & 15] = x;
107 }
108 let f: number;
109 let k: number;
110 if (i < 20) { f = (b & c) | (~b & d); k = 0x5a_82_79_99; }
111 else if (i < 40) { f = b ^ c ^ d; k = 0x6e_d9_eb_a1; }
112 else if (i < 60) { f = (b & c) | (b & d) | (c & d); k = 0x8f_1b_bc_dc | 0; }
113 else { f = b ^ c ^ d; k = 0xca_62_c1_d6 | 0; }
114 const t = (((a << 5) | (a >>> 27)) + f + e + k + x) | 0;
115 e = d; d = c; c = (b << 30) | (b >>> 2); b = a; a = t;
116 }
117 this.h0 = (this.h0 + a) | 0; this.h1 = (this.h1 + b) | 0; this.h2 = (this.h2 + c) | 0; this.h3 = (this.h3 + d) | 0; this.h4 = (this.h4 + e) | 0;
118 }
119 /* oxlint-enable typescript/no-unsafe-type-assertion, typescript/non-nullable-type-assertion-style, unicorn/prefer-math-trunc */
121 // A snapshot: the SHA-1 of the bytes fed so far, and the hasher is left as it was, so digest()/hex() may be called any
122 // number of times and update() continues (a running pack checksum reads the hash so far and keeps hashing). The
123 // padding goes through update(), which moves the five words, bufLen and len; those seven numbers are saved and put
124 // back. The buffered bytes buf[0..bufLen) need no copy: the padding only writes buf at and after bufLen (with fewer
125 // than 56 buffered it fills the block exactly; with more, the second padding block is hashed from `tail` itself).
126 // Before this held, a second digest() hashed the message plus its padding and returned that as a digest.
127 public digest(): Uint8Array {
128 const { h0, h1, h2, h3, h4, bufLen, len } = this;
129 const bits = this.len * 8;
130 const padLen = this.bufLen < 56 ? 56 - this.bufLen : 120 - this.bufLen;
131 const tail = new Uint8Array(padLen + 8);
132 tail[0] = 0x80;
133 const hi = Math.floor(bits / 0x1_00_00_00_00);
134 const lo = bits >>> 0;
135 const dv = new DataView(tail.buffer);
136 dv.setUint32(padLen, hi);
137 dv.setUint32(padLen + 4, lo);
138 this.update(tail);
139 const out = new Uint8Array(20);
140 const odv = new DataView(out.buffer);
141 // setInt32 writes the same four bytes setUint32 wrote for the unsigned word
142 odv.setInt32(0, this.h0); odv.setInt32(4, this.h1); odv.setInt32(8, this.h2); odv.setInt32(12, this.h3); odv.setInt32(16, this.h4);
143 this.h0 = h0; this.h1 = h1; this.h2 = h2; this.h3 = h3; this.h4 = h4;
144 this.bufLen = bufLen;
145 this.len = len;
146 return out;
147 }
149 // A digest is OID_BYTES bytes: its hex is an Oid (an object id, or a pack trailer, which is the same shape).
150 public hex(): Oid { return digestOid(toHex(this.digest())); }
154export type StreamingSha1 =
155 | { readonly kind: 'digeststream'; readonly update: (b: Uint8Array) => Promise<void>; readonly hex: () => Promise<Oid> }
156 | { readonly kind: 'js'; readonly update: (b: Uint8Array) => void; readonly hex: () => Promise<Oid> };
158// Streaming hasher with a native fast path: Workers expose crypto.DigestStream (a WritableStream that digests
159// incrementally at native speed); node and browsers do not, so the pure-JS Sha1 above is the fallback. Same interface,
160// async because the native one is. SHA-1 is ~80% of the streamed-edit CPU on node (test/calibrate.mjs); on Workers the
161// large-read wall is dominated by DecompressionStream (git.blob.inflate), not DigestStream — keep OID verification
162// (pack trailer ≠ content oid), keep DigestStream over buffering crypto.subtle.digest, and measure hash_ms with
163// performance.now (Date.now stamped 0 on Workers). Bench: test/bench-sha1-hashers.mjs.
164export function streamingSha1(): StreamingSha1 {
165 const native = flatMapOption(field(globalThis, 'crypto'), c => field(c, 'DigestStream'));
166 if (native.tag === 'some' && isDigestStream(native.value)) {
167 const ds = new native.value('SHA-1'); // it IS a WritableStream; `digest` is a promise of the ArrayBuffer
168 const w = ds.getWriter();
169 // hex() ends the native stream (it cannot be read without closing it), once: a second hex() is the same promise,
170 // not a second close (which rejects). Unlike Sha1, an update after hex() is not possible here; it rejects.
171 let end: { readonly tag: 'open' } | { readonly tag: 'closed'; readonly oid: Promise<Oid> } = { tag: 'open' };
172 const finish = async (): Promise<Oid> => { await w.close(); return digestOid(toHex(new Uint8Array(await ds.digest))); };
173 // oxlint-disable-next-line typescript/promise-function-async -- hands back the one stored promise
174 const hex = (): Promise<Oid> => {
175 if (end.tag === 'open') end = { tag: 'closed', oid: finish() };
176 return end.oid;
177 };
178 // update hands back the writer's own promise (no extra tick per chunk)
179 // oxlint-disable-next-line typescript/promise-function-async -- see above
180 return { kind: 'digeststream', update: b => w.write(b), hex };
181 }
182 const h = new Sha1();
183 // async to match the native hasher's interface
184 // oxlint-disable-next-line eslint/require-await, typescript/require-await -- see above
185 return { kind: 'js', update: b => { h.update(b); }, hex: async () => h.hex() };
Diagnostics6 git calls · 79 ms
CallStatusSizeTime079 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02002.4 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=70
git.response.bytes=2507
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2507
git.pkt.payload.max=2480
git.sideband.band1.bytes=2480
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=2480
git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381
Open pack map
fetch2004.2 KiB71 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=4309
git.response.cache=MISS
git.response.duration_ms=71
git.fetch.wants=7418d1b7d6dc0b661a08ad6f1323ed913fb776ba
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4309
git.pkt.payload.max=4282
git.sideband.band1.bytes=4282
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2bc12ad61cfc3
Request
b25405e6…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format