GitHubivarvong
ply
main
1// The streamed pack writer push sends (#172). The pack is deflated twice and never held:
2// measurePack (pass 1) deflates every object, keeping only its length, its SHA-1 and each entry's numbers, so a pack
3// over the cap is refused before the first request byte (push.ts, I36);
4// streamPack (pass 2) deflates them again into the request body as it is read, and ends the body with the trailer
5// only when this pass gave the same length and SHA-1 as the first.
6// A pack no longer than INFLATE_CHUNK is kept from pass 1 and sent as bytes (no second pass). Either way the writer holds,
7// beside the caller's objects and zlib's own state, the deflate output of at most one DEFLATE_SLICE slice: the slice and
8// the one block of earlier input zlib may still be holding (level 6 ends a block every 16 Ki symbols), so a push whose
9// largest object is L peaks at about L + DEFLATE_SLICE + 16 KiB (81,940 B measured on random input, node and workerd;
10// 32,776 B with the 16 KiB slices before).
11// Both passes feed CompressionStream the same input in the same DEFLATE_SLICE slices, so zlib (deterministic for a fixed
12// input sequence and parameters) writes the same bytes (test/packwrite.test.js, node and workerd). If it ever did not,
13// pass 2 errors its stream before the trailer: the server is never handed a pack that is whole, and push reports it.
14// The bytes are buildPack's (pack.ts) for the same objects: the same headers, bases and zlib streams.
15import { streamingSha1, fromHex, toHex } from './sha1.js';
16import { builtEntry, entryHead, packHeader } from './pack.js';
17import type { BuiltEntry, PackInput } from './pack.js';
18import { bufferSource, concat } from './bytes.js';
19import { plyError } from './errors.js';
20import type { ErrorOf, PlyError } from './errors.js';
21import { limitExceeded } from './limits.js';
22import { DEFLATE_SLICE, INFLATE_CHUNK, PACK_HEADER_BYTES, PACK_TRAILER_BYTES } from './format.js';
24// One object's zlib stream, pulled: the object goes into CompressionStream in DEFLATE_SLICE slices as the output is read
25// (a source with no read-ahead), so nothing but zlib's own state and one output chunk is held.
26// No read-ahead of slices.
27// Stryker disable next-line ObjectLiteral: equivalent. A slice is a view of the caller's object (subarray, no copy), so a source queue of one costs no bytes, and CompressionStream gets the same slices in the same order
28const SLICE_QUEUE: QueuingStrategy<Uint8Array<ArrayBuffer>> = { highWaterMark: 0 };
29const slices = (data: Uint8Array): ReadableStream<Uint8Array<ArrayBuffer>> => {
30 const bytes = bufferSource(data);
31 let next = 0;
32 return new ReadableStream<Uint8Array<ArrayBuffer>>({
33 pull(c): void {
34 if (next >= bytes.length) { c.close(); return; }
35 const end = Math.min(next + DEFLATE_SLICE, bytes.length);
36 c.enqueue(bytes.subarray(next, end));
37 next = end;
38 },
39 }, SLICE_QUEUE);
40};
41export async function* deflatePull(data: Uint8Array): AsyncGenerator<Uint8Array, void, unknown> {
42 const reader = slices(data).pipeThrough(new CompressionStream('deflate')).getReader();
43 try {
44 for (;;) {
45 // oxlint-disable-next-line no-await-in-loop -- a pull: each chunk is read when the consumer asks for it
46 const r = await reader.read();
47 if (r.done) return;
48 yield r.value;
49 }
50 } finally {
51 // a consumer that stopped early (pass 2's body cancelled) stops the deflate; after the end this is a no-op
52 await reader.cancel();
53 }
54}
56// What one pass wrote: every entry, the bytes before the trailer, and their SHA-1 (the trailer).
57type Tally = { readonly entries: readonly BuiltEntry[]; readonly body: number; readonly digest: Uint8Array };
58// The pack before its trailer, piece by piece (the header, each object's header and base, its zlib chunks), each
59// hashed as it is yielded. The headers and entries are buildPack's (pack.ts entryHead, builtEntry). The hash is
60// streamingSha1's: Workers' native DigestStream where there is one (each pass hashes the whole pack, so the pure-JS
61// SHA-1 was a pass's largest cost after zlib), the pure-JS Sha1 elsewhere.
62async function* pieces(objects: readonly PackInput[]): AsyncGenerator<Uint8Array, Tally, unknown> {
63 const sha = streamingSha1();
64 const hdr = packHeader(objects.length);
65 await sha.update(hdr);
66 yield hdr;
67 const entries: BuiltEntry[] = [];
68 const offsets: number[] = [];
69 let offset = PACK_HEADER_BYTES;
70 for (const o of objects) {
71 // buildPack's header for the entry (pack.ts entryHead): the same type, header and base bytes
72 const h = entryHead(o, offset, offsets);
73 // oxlint-disable-next-line no-await-in-loop -- in pack order
74 await sha.update(h.head);
75 yield h.head;
76 let compressed = 0;
77 // one object at a time, in pack order
78 // oxlint-disable-next-line no-await-in-loop -- see above
79 for await (const z of deflatePull(o.data)) { await sha.update(z); compressed += z.length; yield z; }
80 const entry = builtEntry(h, o.data.length, compressed, offset);
81 entries.push(entry);
82 offsets.push(offset);
83 offset += entry.bytes;
84 }
85 return { entries, body: offset, digest: fromHex(await sha.hex()) };
86}
88// Pass 1. `bytes` is the whole pack (trailer included), what the server would receive; `held` is the pack itself when
89// it is at most `keep` bytes (null otherwise: pass 2 writes it).
90export type MeasuredPack = {
91 readonly objects: readonly PackInput[]; readonly entries: readonly BuiltEntry[]; readonly bytes: number;
92 readonly body: number; readonly digest: Uint8Array; readonly held: Uint8Array | null;
93};
94export async function measurePack(objects: readonly PackInput[], keep: number = INFLATE_CHUNK): Promise<MeasuredPack> {
95 const it = pieces(objects);
96 // the pieces while the pack still fits in `keep` (n only grows: once past it, nothing more is kept)
97 const kept: Uint8Array[] = [];
98 let n = 0;
99 for (;;) {
100 // oxlint-disable-next-line no-await-in-loop -- pass 1 reads its own pieces in order
101 const r = await it.next();
102 if (r.done === true) {
103 const { entries, body, digest } = r.value;
104 const bytes = body + PACK_TRAILER_BYTES;
105 return { objects, entries, bytes, body, digest, held: bytes <= keep ? concat([...kept, digest]) : null };
106 }
107 n += r.value.length;
108 if (n + PACK_TRAILER_BYTES <= keep) kept.push(r.value);
109 }
110}
112// Pass 2: `prefix` (the commands, and push options) and then the pack, as a stream read at the reader's pace. `failure`
113// is null until this pass refuses to finish the pack (a length or SHA-1 that is not pass 1's) or the deflate fails; the
114// stream is then errored before the trailer, whatever the runtime does with an errored request body (undici aborts the
115// request; workerd ends it as if complete, measured), so the server never gets a whole pack. `pulled` is whether the
116// transport ever asked the body for a byte: until it has, nothing of this push left the process (the first pull hands
117// over the command list).
118export type StreamedPack = { readonly stream: ReadableStream<Uint8Array>; readonly length: number; readonly failure: () => ErrorOf<'internal'> | null; readonly pulled: () => boolean };
119export function streamPack(m: MeasuredPack, prefix: Uint8Array): StreamedPack {
120 const it = pieces(m.objects);
121 let failed: ErrorOf<'internal'> | null = null;
122 let started = false;
123 let sent = 0;
124 const refuse = (c: ReadableStreamDefaultController<Uint8Array>, why: string, cause: unknown = null): void => {
125 failed = plyError('internal', `push: the pack's second deflate pass ${why}; it was not sent whole`, { what: 'push pack', got: sent, max: m.body, ...(cause === null ? {} : { cause }) });
126 c.error(failed);
127 };
128 const stream = new ReadableStream<Uint8Array>({
129 async pull(c): Promise<void> {
130 if (!started) { started = true; c.enqueue(prefix); return; }
131 let r: IteratorResult<Uint8Array, Tally>;
132 try { r = await it.next(); } catch (e) { refuse(c, 'failed', e); return; }
133 if (r.done === true) {
134 if (r.value.body !== m.body) { refuse(c, 'ran shorter than the first'); return; }
135 if (toHex(r.value.digest) !== toHex(m.digest)) { refuse(c, 'wrote another pack'); return; }
136 c.enqueue(r.value.digest);
137 c.close();
138 return;
139 }
140 sent += r.value.length;
141 // never more than pass 1 measured: the cap push checked is the cap the wire sees
142 if (sent > m.body) { refuse(c, 'ran longer than the first'); return; }
143 c.enqueue(r.value);
144 },
145 // the value handed to return() becomes the finished generator's result, which nothing reads
146 // Stryker disable next-line ObjectLiteral,ArrayDeclaration: equivalent, see above
147 async cancel(): Promise<void> { await it.return({ entries: [], body: 0, digest: new Uint8Array(0) }); },
148 }, { highWaterMark: 0 });
149 return { stream, length: prefix.length + m.bytes, failure: () => failed, pulled: () => started };
150}
152// A pack the caller already has, as the request body after `prefix` (commands, and push options): read at the reader's
153// pace, never buffered or teed, one chunk held back. `length` is the pack's own length when the caller knows it (null:
154// it does not); SourcedPack.length is the whole body, the prefix and that pack (only the prefix when unknown: the real
155// count is `sent()`, for the span).
156// The last chunk is released only once the source has ended and its length has passed, and a chunk that would go over
157// `length` or `maxPack` is never released, so a refused pack never reaches the server whole, whatever the runtime does
158// with an errored request body (undici aborts the request; workerd ends it as if complete, measured: streamPack above).
159// `failure` is null until the pack is refused or the source fails; `release` cancels a source the request did not finish
160// (asked once, not waited for).
161// `pulled` is whether the transport ever asked the body for a byte. Until it has, nothing of this push left the
162// process (the first pull hands over the command list), and `giveBack` then returns the source as it came: unlocked,
163// unread and not cancelled (after a pull it does nothing). A body given back errors if it is pulled after all, so it
164// sends nothing. A transport that cancels the body itself still cancels the source, pulled or not.
165export type SourcedPack = {
166 readonly stream: ReadableStream<Uint8Array>; readonly length: number;
167 readonly failure: () => PlyError | null; readonly sent: () => number; readonly release: () => Promise<void>;
168 readonly pulled: () => boolean; readonly giveBack: () => void;
169};
170export function streamSource(source: ReadableStream<Uint8Array>, prefix: Uint8Array, length: number | null, maxPack: number): SourcedPack {
171 const reader = source.getReader();
172 let failed: PlyError | null = null;
173 let started = false;
174 let open = true;
175 let held: Uint8Array | null = null;
176 let read = 0;
177 let sent = 0;
178 // The source is the caller's: it is asked to cancel and never waited for, so a cancel() that never settles cannot hold
179 // the push past its outcome (its report, its abort, its timeout). A source that fails to cancel changes nothing.
180 // oxlint-disable-next-line eslint/require-await -- async for the callers that await it; nothing here waits
181 const release = async (why: unknown = null): Promise<void> => {
182 if (!open) return;
183 open = false;
184 // a cancel() that throws where it stands, or answers with something that is not a promise, changes nothing either
185 try { Promise.resolve(reader.cancel(why)).catch(() => null); } catch { /* the source's own failure to cancel: nothing waits on it */ }
186 };
187 const giveBack = (): void => {
188 if (!open || started) return;
189 open = false;
190 reader.releaseLock();
191 };
192 const refuse = async (c: ReadableStreamDefaultController<Uint8Array>, e: PlyError): Promise<void> => { failed = e; c.error(e); await release(e); };
193 const mismatch = (want: number): ErrorOf<'bad_request'> => plyError('bad_request', `pushPack: the pack stream yielded ${read} bytes where length said ${want}; it was not sent whole`, { what: 'push pack length', got: read, max: want });
194 const emit = (c: ReadableStreamDefaultController<Uint8Array>, chunk: Uint8Array): void => { sent += chunk.length; c.enqueue(chunk); };
195 const stream = new ReadableStream<Uint8Array>({
196 async pull(c): Promise<void> {
197 if (!started) {
198 if (!open) { failed = plyError('internal', 'pushPack: the request body was pulled after the push had ended; nothing was sent', { what: 'push pack' }); c.error(failed); return; }
199 started = true; c.enqueue(prefix); return;
200 }
201 for (;;) {
202 let r: ReadableStreamReadResult<Uint8Array>;
203 // oxlint-disable-next-line no-await-in-loop -- one chunk at a time, at the request's pace
204 try { r = await reader.read(); } catch (e) { open = false; await refuse(c, plyError('internal', 'pushPack: the pack stream failed; it was not sent whole', { what: 'push pack', got: read, cause: e })); return; }
205 if (r.done) {
206 open = false;
207 // oxlint-disable-next-line no-await-in-loop -- this refusal ends the loop
208 if (length !== null && read !== length) { await refuse(c, mismatch(length)); return; }
209 if (held !== null) emit(c, held);
210 c.close();
211 return;
212 }
213 const chunk: unknown = r.value;
214 // oxlint-disable-next-line no-await-in-loop -- each refusal ends the loop
215 if (!(chunk instanceof Uint8Array)) { await refuse(c, plyError('bad_request', 'pushPack: the pack stream yielded something other than bytes', { what: 'push pack chunk' })); return; }
216 read += chunk.length;
217 // oxlint-disable-next-line no-await-in-loop -- see above
218 if (read > maxPack) { await refuse(c, limitExceeded('push pack bytes', read, maxPack)); return; }
219 // oxlint-disable-next-line no-await-in-loop -- see above
220 if (length !== null && read > length) { await refuse(c, mismatch(length)); return; }
221 if (chunk.length > 0) {
222 const out = held;
223 held = chunk;
224 if (out !== null) { emit(c, out); return; }
225 }
226 }
227 },
228 async cancel(why): Promise<void> { await release(why); },
229 }, { highWaterMark: 0 });
230 return { stream, length: prefix.length + (length === null ? 0 : length), failure: () => failed, sent: () => sent, release, pulled: () => started, giveBack };
231}
Diagnostics6 git calls · 130 ms
CallStatusSizeTime0130 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=11990 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02002.4 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=122 git.response.bytes=2507 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2507 git.pkt.payload.max=2480 git.sideband.band1.bytes=2480 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=2480 git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381Open pack map
fetch2004.7 KiB122 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=4808 git.response.cache=MISS git.response.duration_ms=122 git.fetch.wants=d12d5f653c232d860b46a44bfc8a01803f7d35a4 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4808 git.pkt.payload.max=4781 git.sideband.band1.bytes=4781 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e0f83a0a68332c
- Request
- 088a8cea…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format