GitHubivarvong
ply
main
1// The ply pipeline running inside workerd. POST /edit with {url, path, find, replace, n?} runs editFile n times
2// (sequentially, or concurrently with ?concurrent=1) and reports results, spans summary, and timing.
3import { editFile } from '../../src/edit.js';
4import { readFile } from '../../src/read.js';
5import { unwrapAsync, unwrap } from '../unwrap.js';
6import { Tracer, arrayExporter, flattenLine } from '../../src/otel.js';
7import { GitHttp } from '../../src/client.js';
8import { withLimits } from '../../src/limits.js';
9import { cachingFetch, memoryCache } from '../../src/packcache.js';
10const packMemory = memoryCache();
11import { generateSigner } from '../../src/sign.js';
12import { createTransport } from '../../src/http.js';
13import { push, ZERO_OID } from '../../src/push.js';
14import { pushPack } from '../../src/receivecaps.js';
15import { hashObject, encodeTree, encodeCommit, TYPE } from '../../src/objects.js';
16import { measurePack, streamPack } from '../../src/packwrite.js';
17import { buildPack } from '../../src/pack.js';
18import { packSubset } from '../../src/packsubset.js';
19import { scanPackIndex } from '../../src/packscanindex.js';
20import { failure, fetchOps } from '../sweep/ops.mjs';
22// module-scoped in-memory pack cache for workerd tests (never caches.default — Workers Cache is ply-web).
23const packCache = packMemory;
25export default {
26 async fetch(req, env, ctx) {
27 const u = new URL(req.url);
28 if (u.pathname === '/ping') return new Response('pong');
29 if (u.pathname === '/features') {
30 const f = {};
31 f.DecompressionStream = typeof DecompressionStream;
32 f.CompressionStream = typeof CompressionStream;
33 f.AbortSignalAny = typeof AbortSignal.any;
34 f.AbortSignalTimeout = typeof AbortSignal.timeout;
35 f.getRandomValues = typeof crypto.getRandomValues;
36 f.subtle = typeof crypto.subtle;
37 f.performanceNow = typeof performance?.now;
38 f.dateNowAdvances = (() => { const a = Date.now(); let x = 0; for (let i = 0; i < 1e6; i++) x += i; return Date.now() - a; })();
39 try { const ds = new DecompressionStream('deflate'); f.deflateOk = !!ds; } catch (e) { f.deflateOk = e.message; }
40 return Response.json(f);
41 }
42 if (u.pathname === '/fetchpack') {
43 const body = await req.json();
44 const exporter = arrayExporter();
45 const c = unwrap(GitHttp.create(body.url, { tracer: new Tracer({ exporter }) }));
46 const t0 = Date.now();
47 try {
48 const r = await unwrapAsync(c.fetchPack(body.req));
49 return Response.json({ ok: true, objects: r.parsed?.stats.count ?? 0, ms: Date.now() - t0, spans: exporter.lines.flatMap(flattenLine).map(s => ({ name: s.name, ms: s.durationMs, attrs: pick(s.attrs) })) });
50 } catch (e) { return Response.json({ ok: false, code: e.code, message: e.message, ms: Date.now() - t0 }); }
51 }
52 if (u.pathname === '/redirect') { // ply against a redirecting server; plus what workerd's own `follow` does with Authorization
53 const body = await req.json();
54 const c = unwrap(GitHttp.create(body.url, { tracer: new Tracer(), auth: body.auth }));
55 const res = {};
56 for (const [k, op] of [['get', () => unwrapAsync(c.discover())], ['post', () => unwrapAsync(c.lsRefs())]]) { try { await op(); res[k] = { ok: true }; } catch (e) { res[k] = { code: e.code, status: e.status, message: e.message }; } }
57 try { const r = await fetch(`${body.url}/info/refs?service=git-upload-pack`, { headers: { Authorization: 'Bearer workerd-follow-probe' } }); res.follow = { status: r.status, url: r.url }; await r.body?.cancel(); } catch (e) { res.follow = { error: e.message }; }
58 return Response.json(res);
59 }
60 if (u.pathname === '/read') { // readFile's blob stream, drained or cancelled after one chunk: its follows span (#35)
61 const body = await req.json();
62 const exporter = arrayExporter();
63 try {
64 const r = await unwrapAsync(readFile(body.url, { path: body.path }, { tracer: new Tracer({ exporter }), ...(body.limits ? { limits: body.limits } : {}) }));
65 let bytes = 0;
66 const rd = r.stream.getReader();
67 if (body.cancel) { const { value } = await rd.read(); bytes = value?.length ?? 0; await rd.cancel('enough'); }
68 else for (;;) { const { value, done } = await rd.read(); if (done) break; bytes += value.length; }
69 return Response.json({ ok: true, size: r.size, bytes, spans: exporter.lines.flatMap(flattenLine).map(s => ({ name: s.name, traceId: s.traceId, spanId: s.spanId, parentSpanId: s.parentSpanId ?? null, links: s.links.map(l => [l.spanId, l.attrs['ply.link']]), outcome: s.attrs[`${s.name}.outcome`] ?? null })) });
70 } catch (e) { return Response.json({ ok: false, code: e.code, what: e.what ?? null, message: e.message, blob: exporter.lines.flatMap(flattenLine).filter(s => s.name === 'git.read.blob').map(s => s.attrs['git.read.blob.outcome'] ?? null) }); }
71 }
72 if (u.pathname === '/sweep') { // one limit-sweep cell (test/sweep): the same operation code the node cell runs
73 const cell = await req.json();
74 const op = fetchOps(cell, { tracer: new Tracer(), limits: cell.limits ?? {} })[cell.op];
75 if (!op) return Response.json({ status: 'crash', code: null, msg: `workerd: no op ${cell.op}` });
76 try { return Response.json({ status: 'ok', ...(await op()) }); } catch (e) { return Response.json(failure(e)); }
77 }
78 if (u.pathname === '/push') { // a streamed push (#172): an L-byte random blob; mutate: the blob changes between the passes
79 const body = await req.json();
80 const blob = new Uint8Array(body.size);
81 for (let i = 0; i < blob.length; i += 65536) crypto.getRandomValues(blob.subarray(i, Math.min(i + 65536, blob.length)));
82 const tree = unwrap(encodeTree([{ mode: '100644', name: 'big.bin', oid: hashObject('blob', blob) }]));
83 const commit = unwrap(encodeCommit({ tree: hashObject('tree', tree), author: 'a <a@x> 1700000000 +0000', message: 'm\n' }));
84 const oid = hashObject('commit', commit);
85 const objects = [{ type: TYPE.BLOB, data: blob }, { type: TYPE.TREE, data: tree }, { type: TYPE.COMMIT, data: commit }];
86 const seen = [];
87 const fetchImpl = (url, init) => {
88 if (String(url).endsWith('/git-receive-pack')) { seen.push({ streamed: typeof init.body?.getReader === 'function', duplex: init.duplex ?? null, redirect: init.redirect }); if (body.mutate) blob[100] ^= 1; }
89 return fetch(url, init);
90 };
91 const t = unwrap(createTransport(body.url, { fetchImpl, limits: withLimits({ maxPackBytes: 4 * body.size + 4096 }) }));
92 const r = await push(t, { ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: oid, objects });
93 return Response.json({ ok: r.ok, code: r.ok ? null : r.error.code, original: r.ok ? null : r.error.original ?? null, message: r.ok ? null : r.error.message, commit: oid, packBytes: r.ok ? r.value.packBytes : null, seen });
94 }
95 if (u.pathname === '/push-pack') { // a pack as a stream (#234). mode: ok; short (length says one more); errored (the source fails partway)
96 const body = await req.json();
97 const blob = new Uint8Array(body.size);
98 for (let i = 0; i < blob.length; i += 65536) crypto.getRandomValues(blob.subarray(i, Math.min(i + 65536, blob.length)));
99 const tree = unwrap(encodeTree([{ mode: '100644', name: 'big.bin', oid: hashObject('blob', blob) }]));
100 const commit = unwrap(encodeCommit({ tree: hashObject('tree', tree), author: 'a <a@x> 1700000000 +0000', message: 'm\n' }));
101 const oid = hashObject('commit', commit);
102 const whole = (await measurePack([{ type: TYPE.BLOB, data: blob }, { type: TYPE.TREE, data: tree }, { type: TYPE.COMMIT, data: commit }], 4 * body.size + 4096)).held;
103 const CHUNK = 16384;
104 let at = 0;
105 const pack = new ReadableStream({
106 pull(c) {
107 if (body.mode === 'errored' && at >= whole.length >> 1) { c.error(new Error('source died')); return; }
108 if (at >= whole.length) { c.close(); return; }
109 c.enqueue(whole.slice(at, at + CHUNK));
110 at += CHUNK;
111 },
112 }, { highWaterMark: 0 });
113 const seen = [];
114 const fetchImpl = (url, init) => {
115 if (String(url).endsWith('/git-receive-pack')) seen.push({ streamed: typeof init.body?.getReader === 'function', duplex: init.duplex ?? null, redirect: init.redirect });
116 return fetch(url, init);
117 };
118 const t = unwrap(createTransport(body.url, { fetchImpl, limits: withLimits({ maxPackBytes: 4 * body.size + 4096 }) }));
119 const r = await pushPack(t, { ref: 'refs/heads/main', oldOid: ZERO_OID, newOid: oid, pack, length: body.mode === 'short' ? whole.length + 1 : whole.length });
120 return Response.json({ ok: r.ok, code: r.ok ? null : r.error.code, original: r.ok ? null : r.error.original ?? null, message: r.ok ? null : r.error.message, commit: oid, packBytes: r.ok ? r.value.packBytes : null, whole: whole.length, seen });
121 }
122 if (u.pathname === '/cancel-pending-read') { // packSubset's chunk path: the platform's reader on an internal stream (a fetch body, as an R2 body is)
123 const body = await req.json();
124 const res = await fetch(body.url);
125 const reader = ReadableStream.prototype.getReader.call(res.body);
126 const unhandled = [];
127 const onRejection = e => { unhandled.push(String(e.reason?.stack ?? e.reason).slice(0, 300)); };
128 addEventListener('unhandledrejection', onRejection);
129 let got = 0;
130 while (got < body.bytes) { const r = await reader.read(); if (r.done) break; got += r.value.length; }
131 // the server has sent all it will: this read is in flight, and stays so for 20 turns
132 let state = 'pending';
133 reader.read().then(r => { state = r.done ? 'done' : 'value'; }, () => { state = 'rejected'; });
134 for (let i = 0; i < 20; i++) await new Promise(r => setTimeout(r, 0));
135 const before = state;
136 reader.cancel(new Error('stop')).catch(() => null);
137 let turns = 0;
138 for (; turns < 200 && state === 'pending'; turns++) await new Promise(r => setTimeout(r, 0));
139 for (let i = 0; i < 20; i++) await new Promise(r => setTimeout(r, 0));
140 removeEventListener('unhandledrejection', onRejection);
141 return Response.json({ got, before, after: state, turns, unhandled });
142 }
143 if (u.pathname === '/subset-cancel') { // packSubset over a range that is an internal stream (a fetch body that stalls), cancelled mid-read
144 const body = await req.json();
145 const turn = () => new Promise(r => setTimeout(r, 0));
146 const data = new Uint8Array(65536);
147 crypto.getRandomValues(data);
148 const pack = (await buildPack([{ type: TYPE.BLOB, data }])).pack;
149 const idx = unwrap(await scanPackIndex(new Response(pack).body));
150 const rejections = [];
151 const onRejection = e => { rejections.push(String(e.reason?.stack ?? e.reason).slice(0, 600)); };
152 addEventListener('unhandledrejection', onRejection);
153 const source = { size: pack.length, read: async () => ({ ok: true, value: (await fetch(body.url)).body }) };
154 const reader = unwrap(await packSubset(source, idx, [0])).stream.getReader();
155 let got = 0;
156 while (got < body.bytes) { const r = await reader.read(); if (r.done) break; got += r.value.length; }
157 let read = 'pending';
158 reader.read().then(r => { read = r.done ? 'done' : 'value'; }, () => { read = 'rejected'; });
159 for (let i = 0; i < 20; i++) await turn();
160 const before = read;
161 let cancel = 'pending';
162 reader.cancel(new Error('stop')).then(() => { cancel = 'settled'; }, () => { cancel = 'rejected'; });
163 for (let i = 0; i < 200 && (read === 'pending' || cancel === 'pending'); i++) await turn();
164 for (let i = 0; i < 20; i++) await turn(); // a rejection nobody handled is reported by now
165 removeEventListener('unhandledrejection', onRejection);
166 return Response.json({ got, before, read, cancel, rejections });
167 }
168 if (u.pathname === '/timeouts') { // #255 under workerd: {url, mode, limits, holdMs}; frames (the consumer holds the first one holdMs, #273) over a POST whose server drips, stalls or is slow
169 const body = await req.json();
170 const t = unwrap(createTransport(body.url, { limits: withLimits(body.limits) }));
171 const span = { attrs: {}, set(a) { Object.assign(this.attrs, a); }, event() {}, error() {} };
172 let upload = new Uint8Array(4);
173 if (body.mode === 'uploadstall' || body.mode === 'uploadsteady') {
174 let i = 0;
175 const n = 12;
176 const stream = new ReadableStream({
177 async pull(c) {
178 if (body.mode === 'uploadstall' && i >= 2) { await new Promise(() => {}); return; }
179 if (i >= n) { c.close(); return; }
180 if (i > 0) await new Promise(r => setTimeout(r, 40));
181 i++; c.enqueue(new Uint8Array(1024));
182 },
183 }, { highWaterMark: 0 });
184 upload = { stream, length: n * 1024 };
185 }
186 const t0 = Date.now();
187 let frames = 0;
188 try {
189 for await (const _ of t.frames('git-upload-pack', upload, span, { bypassCache: false })) { if (frames++ === 0 && body.holdMs > 0) await new Promise(r => setTimeout(r, body.holdMs)); }
190 return Response.json({ ok: true, frames, ms: Date.now() - t0, limit: span.attrs['git.timeout.limit'] ?? null });
191 } catch (e) {
192 return Response.json({ ok: false, code: e.code, what: e.what ?? null, message: e.message, ms: Date.now() - t0, limit: span.attrs['git.timeout.limit'] ?? null });
193 }
194 }
195 if (u.pathname === '/packwrite') { // both passes and buildPack agree byte for byte under workerd's CompressionStream
196 const body = await req.json();
197 const out = [];
198 for (const size of body.sizes) {
199 const data = new Uint8Array(size);
200 for (let i = 0; i < size; i += 65536) crypto.getRandomValues(data.subarray(i, Math.min(i + 65536, size)));
201 if (body.half) data.fill(0, size >> 1);
202 const objects = [{ type: TYPE.BLOB, data }, { type: TYPE.BLOB, data: data.subarray(0, size >> 2) }];
203 const built = (await buildPack(objects)).pack;
204 const m = await measurePack(objects, 0);
205 const a = new Uint8Array(await new Response(streamPack(m, new Uint8Array(0)).stream).arrayBuffer());
206 const b = new Uint8Array(await new Response(streamPack(m, new Uint8Array(0)).stream).arrayBuffer());
207 const same = (x, y) => x.length === y.length && x.every((v, i) => v === y[i]);
208 out.push({ size, bytes: m.bytes, passes: same(a, b), built: same(a, built) });
209 }
210 return Response.json(out);
211 }
212 if (u.pathname === '/edit') {
213 const body = await req.json();
214 const n = body.n ?? 1, concurrent = u.searchParams.get('concurrent') === '1';
215 const exporter = arrayExporter();
216 const tracer = new Tracer({ exporter });
217 const cache = new Map();
218 const limits = withLimits(body.limits ?? {});
219 const fetchImpl = body.packCache && packCache ? cachingFetch(globalThis.fetch, packCache, { waitUntil: p => ctx.waitUntil(p) }) : globalThis.fetch;
220 const signing = body.sign ? await generateSigner() : null; // Ed25519 via the platform's WebCrypto
221 const one = async i => {
222 const t0 = Date.now();
223 try {
224 const r = await unwrapAsync(editFile(body.url, { path: body.path, find: body.find, replace: body.replace, author: { name: 'wd', email: 'wd@x' }, message: `wd ${i}` }, { tracer, cache, limits, auth: body.auth, fetchImpl, head: body.head ?? null, signer: signing?.signer ?? null }));
225 return { ok: true, ms: Date.now() - t0, newCommit: r.newCommit, oldCommit: r.oldCommit, pushBytes: r.pushBytes, noop: !!r.noop, hint: r.hint, packCache: fetchImpl.stats ?? null };
226 } catch (e) { return { ok: false, ms: Date.now() - t0, code: e.code, message: e.message }; }
227 };
228 const results = concurrent ? await Promise.all(Array.from({ length: n }, (_, i) => one(i))) : [];
229 if (!concurrent) for (let i = 0; i < n; i++) results.push(await one(i));
230 const spans = exporter.lines.flatMap(flattenLine);
231 const root = spans.filter(s => s.name === 'git.edit');
232 return Response.json({ signer: signing?.signer.publicKeyLine ?? null, results, roots: root.map(s => pick(s.attrs)), blob: spans.filter(s => s.name === 'git.edit.blob').map(s => pick(s.attrs)), spanCount: spans.length, ...(body.dump ? { dump: JSON.stringify(exporter.lines) } : {}) });
233 }
236 return new Response('not found', { status: 404 });
237 },
238};
239const pick = a => Object.fromEntries(Object.entries(a).filter(([k]) => /outcome|mem\.|bytes|ms$|objects|cache_hits|match|ratio|status_code|pack\.cache|head\.hint|head\.outcome|attempt|hasher|cpu/.test(k)));
Diagnostics7 git calls · 234 ms
CallStatusSizeTime0234 ms
discover200191 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB101 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=94 git.response.bytes=11990 git.response.cache=EXPIRED git.response.duration_ms=101 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetchtree:0200139 B · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=139 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=a0bbad5eee2bb61cd47bbeeda697503b78f8cc38 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=139 git.pkt.payload.max=112 git.sideband.band1.bytes=112 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=112 git.pack.trailer=5ef942969b42f0d05b3f14b041f4c1749c0753ceOpen pack map
fetch2004.8 KiB122 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=122 git.response.bytes=4916 git.response.cache=MISS git.response.duration_ms=122 git.fetch.wants=0d127333e9441cb2e4df7da3ea800a13ada06817 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4916 git.pkt.payload.max=4889 git.sideband.band1.bytes=4889 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e16f8d9dc86102
- Request
- bc9a5007…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format