main
1// Async-iterable adapters over the pull decoders (#241): scanPack (packstream.ts), pktFrames (pktline.ts) and
2// inflateChunks (inflatestream.ts). The pull API stays the base; these are the read loop every streaming caller wrote
3// around it, once:
4//
5// for await (const piece of scanPack(response.body, { limits })) { ... }
6//
7// - The adapter reads its source only when the decoder answers `need`: one chunk per need, nothing ahead, so a slow
8// consumer is the source's backpressure. What it holds is the decoder's hold (streambound.ts) and the one chunk fed.
9// - Each returns a ReadableStream of the decoder's items (async-iterable on node and workerd), with a high-water mark of
10// 0. It feeds, closes the decoder when the source ends, and closes at the decoder's end. `need` and `end` are never
11// items.
12// - A decoder refusal errors the stream with the decoder's typed PlyError (the value its pull returns) and cancels the
13// source: results never throw, streams error, so `for await` throws it and a reader's read() rejects with it. A source
14// that errors errors the stream with its own error; a chunk that is not a Uint8Array is `bad_request`.
15// - Cancelling (a `break` or a throw in a `for await` body, reader.cancel()) cancels the source: a ReadableStream's
16// reader is cancelled (its underlying source sees cancel(reason)), an AsyncIterable's iterator gets return().
17// - A pack chunk and an inflate chunk are the caller's (copies); a pkt-line payload is a copy too (pktStep).
18import type { ErrorOf } from './errors.js';
19import { plyError } from './errors.js';
20import type { Option, Result } from './result.js';
21import { ok, err, some, none, assertNever } from './result.js';
22import { withDefaults, at as itemAt } from './absent.js';
23import type { LimitsInput } from './limits.js';
24import { DEFAULT_LIMITS } from './limits.js';
25import type { PackError, PackPiece, ScanBudget } from './packstream.js';
26import { RETAIN_BUDGET, parsePackInit, parsePackFeed, parsePackClose, parsePackPull } from './packstream.js';
27import type { PktFrame, PktState } from './pktline.js';
28import { pktInit, pktStep, pktFinish } from './pktline.js';
29import type { InflateError, InflateStreamOpts } from './inflatestream.js';
30import { inflateInit, inflateFeed, inflatePull, inflateEnd, inflateUnread, cappedBound, zlibWrap } from './inflatestream.js';
32export type ByteSource = ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;
34// One decoder event, as the driver sees it.
35type Event<T> = { readonly kind: 'need' } | { readonly kind: 'item'; readonly value: T } | { readonly kind: 'end' };
36const NEED: Event<never> = Object.freeze({ kind: 'need' });
37const END: Event<never> = Object.freeze({ kind: 'end' });
38const item = <T>(value: T): Event<T> => ({ kind: 'item', value });
39// A pull decoder behind one shape: pull the next event (or its refusal), feed a chunk, close the input.
40type Decoder<T, E> = {
41 readonly pull: () => Result<Event<T>, E>;
42 readonly feed: (bytes: Uint8Array) => void;
43 readonly close: () => void;
44};
46// The source, read a chunk at a time. Its reader is taken on the first read, so an iterator that is never pulled
47// leaves the stream unlocked.
48type Reader = {
49 readonly read: () => Promise<Option<unknown>>;
50 readonly cancel: (why: unknown) => Promise<void>;
51};
52function readerOf(source: ByteSource): Reader {
53 if ('getReader' in source) {
54 let held: Option<ReadableStreamDefaultReader<Uint8Array>> = none;
55 const take = (): ReadableStreamDefaultReader<Uint8Array> => {
56 if (held.tag === 'some') return held.value;
57 const r = source.getReader();
58 held = some(r);
59 return r;
60 };
61 return {
62 read: async () => { const r = await take().read(); return r.done ? none : some(r.value); },
63 // a stream never read is cancelled without a reader; a source whose cancel fails changes nothing for the caller
64 cancel: async why => { await (held.tag === 'some' ? held.value.cancel(why) : source.cancel(why)).catch(() => null); },
65 };
66 }
67 const it = source[Symbol.asyncIterator]();
68 return {
69 read: async () => { const r = await it.next(); return r.done === true ? none : some(r.value); },
70 cancel: async () => { if (typeof it.return === 'function') await Promise.resolve(it.return()).catch(() => null); },
71 };
72}
74type Round = 'settled' | 'reading';
75// The driver, as a stream of the decoder's items with a high-water mark of 0: it pulls only while a read is waiting,
76// so it reads its source one chunk per need and never ahead. A refusal errors the stream (and cancels the source); a
77// source that rejects errors it with its own error; cancel (a `break`) cancels the source.
78function drive<T, E>(source: ByteSource, opened: Result<Decoder<T, E>, E>): ReadableStream<T> {
79 const reader = readerOf(source);
80 // one chunk in: the decoder's input, its close, or a refusal for a chunk that is not bytes
81 const feedFrom = (dec: Decoder<T, E>, got: Option<unknown>): Result<true, ErrorOf<'bad_request'>> => {
82 if (got.tag === 'none') { dec.close(); return ok(true); }
83 if (!(got.value instanceof Uint8Array)) return err(plyError('bad_request', `stream source: a chunk is ${typeof got.value}, not a Uint8Array`));
84 dec.feed(got.value);
85 return ok(true);
86 };
87 // Pull until one item is out or the decoder ended. A need reads one chunk and goes round again from the read's
88 // callback: each round starts afresh, so a long run of needs (a frame fed a byte at a time) holds nothing per round.
89 // A cancel while a read is out stops the round that read would start.
90 let cancelled = false;
91 const deliver = async (dec: Decoder<T, E>, c: ReadableStreamDefaultController<T>): Promise<void> => {
92 await new Promise<void>(settled => {
93 const stop = (why: unknown): void => {
94 c.error(why);
95 reader.cancel(why).then(settled).catch(settled);
96 };
97 const failed = (why: unknown): void => { c.error(why); settled(); };
98 // `input` is the chunk the last need read (none on the first round). The answer says whether this round settled
99 // the pull or left a read out.
100 const round = (input: Option<Option<unknown>>): Round => {
101 if (cancelled) { settled(); return 'settled'; }
102 if (input.tag === 'some') {
103 const r = feedFrom(dec, input.value);
104 if (!r.ok) { stop(r.error); return 'settled'; }
105 }
106 const got = dec.pull();
107 if (!got.ok) { stop(got.error); return 'settled'; }
108 const ev = got.value;
109 switch (ev.kind) {
110 case 'item': c.enqueue(ev.value); settled(); return 'settled';
111 case 'end': c.close(); settled(); return 'settled';
112 case 'need': reader.read().then(chunk => round(some(chunk))).catch(failed); return 'reading';
113 }
114 // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check
115 return assertNever(ev);
116 };
117 round(none);
118 });
119 };
120 return new ReadableStream<T>({
121 async pull(c) {
122 if (!opened.ok) {
123 c.error(opened.error);
124 await reader.cancel(opened.error);
125 return;
126 }
127 await deliver(opened.value, c);
128 },
129 async cancel(why) {
130 cancelled = true;
131 await reader.cancel(why);
132 },
133 }, new CountQueuingStrategy({ highWaterMark: 0 }));
134}
136// ---- scanPack: the pack scan's pieces (packstream.ts), never `need` or `end`.
137export type ScanPiece = Exclude<PackPiece, { readonly kind: 'need' } | { readonly kind: 'end' }>;
138export type ScanPackOpts = { readonly limits: LimitsInput; readonly verify: boolean; readonly budget: ScanBudget };
139const scanDefaults = (): ScanPackOpts => ({ limits: DEFAULT_LIMITS, verify: true, budget: RETAIN_BUDGET });
140export function scanPack(source: ByteSource, input: Partial<ScanPackOpts> = {}): ReadableStream<ScanPiece> {
141 const opts = withDefaults(input, scanDefaults());
142 const st = parsePackInit({ limits: opts.limits, verify: opts.verify, budget: opts.budget });
143 const opened: Result<Decoder<ScanPiece, PackError>, PackError> = st.ok ? ok({
144 pull: () => {
145 const p = parsePackPull(st.value);
146 if (!p.ok) return p;
147 const v = p.value;
148 if (v.kind === 'need') return ok(NEED);
149 return ok(v.kind === 'end' ? END : item(v));
150 },
151 feed: bytes => { parsePackFeed(st.value, bytes); },
152 close: () => { parsePackClose(st.value); },
153 }) : st;
154 return drive(source, opened);
155}
157// ---- pktFrames: pkt-line frames (pktline.ts). The source ending inside a frame is pktFinish's refusal.
158type Protocol = ErrorOf<'protocol'>;
159export function pktFrames(source: ByteSource): ReadableStream<PktFrame> {
160 let st: PktState = pktInit();
161 // the frames of the last chunk not handed out yet, last first: a frame handed out is popped, so it is the caller's
162 // alone (holding it here too would keep one more frame than peakBound('pkt') counts)
163 let ready: PktFrame[] = [];
164 let closed = false;
165 let failed: Option<Protocol> = none;
166 const dec: Decoder<PktFrame, Protocol> = {
167 pull: () => {
168 if (failed.tag === 'some') return err(failed.value);
169 const f = itemAt(ready, ready.length - 1);
170 if (f.tag === 'some') { ready.pop(); return ok(item(f.value)); }
171 if (!closed) return ok(NEED);
172 const fin = pktFinish(st);
173 return fin.ok ? ok(END) : fin;
174 },
175 feed: bytes => {
176 const r = pktStep(st, bytes);
177 if (!r.ok) { failed = some(r.error); return; }
178 st = r.value.state;
179 ready = r.value.frames.toReversed();
180 },
181 close: () => { closed = true; },
182 };
183 return drive(source, ok(dec));
184}
186// ---- inflateChunks: one zlib member's (or raw deflate stream's) output (inflatestream.ts). The source must end with
187// the member: a byte after it is pack_corrupt, as a truncated member is.
188export type InflateChunksOpts = InflateStreamOpts;
189const trailing = (n: number): ErrorOf<'pack_corrupt'> => plyError('pack_corrupt', `inflate: ${n} bytes after the end of the stream`);
190const inflateDefaults = (): InflateChunksOpts => ({ bound: cappedBound(), wrap: zlibWrap(true) });
191export function inflateChunks(source: ByteSource, input: Partial<InflateChunksOpts> = {}): ReadableStream<Uint8Array> {
192 const opts = withDefaults(input, inflateDefaults());
193 const opened = inflateInit(opts);
194 if (!opened.ok) return drive(source, opened);
195 const st = opened.value;
196 let closed = false;
197 // After the member's end the inflater stays at its end and holds what it is fed after (inflateUnread): the next pull
198 // returns 'end' again and reads that as trailing bytes, so a byte after the member is refused whichever chunk brought it.
199 const dec: Decoder<Uint8Array, InflateError> = {
200 pull: () => {
201 const p = closed ? inflateEnd(st) : inflatePull(st);
202 if (!p.ok) return p;
203 switch (p.value.kind) {
204 case 'chunk': return ok(item(p.value.bytes));
205 case 'need': return ok(NEED);
206 case 'end': {
207 const left = inflateUnread(st).length;
208 if (left > 0) return err(trailing(left));
209 return ok(closed ? END : NEED);
210 }
211 }
212 // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check
213 return assertNever(p.value);
214 },
215 feed: bytes => { inflateFeed(st, bytes); },
216 close: () => { closed = true; },
217 };
218 return drive(source, ok(dec));
219}
Diagnostics4 git calls · 242 ms
CallStatusSizeTime0242 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=UPDATING git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2003.8 KiB237 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=237 git.response.bytes=3942 git.response.cache=MISS git.response.duration_ms=237 git.fetch.wants=75190e783c3f1500a11a331259f43f92bb2c4209 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3942 git.pkt.payload.max=3921 git.sideband.band1.bytes=3920 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e234314f26a594
- Request
- 263f00f3…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format