GitHubivarvong
ply
main
1// One sweep cell, in its own process. The parent passes --max-old-space-size when the cell is the 128 MiB heap.
2// argv[2] is the cell JSON. One JSON line on stdout; exit 0 even on a typed refusal so the parent can tell a
3// refusal from a crash. Peak RSS is the kernel's VmHWM for this process (vmHwmKiB below). ArrayBuffers are sampled because they
4// sit outside the V8 heap the flag caps.
5import { readFileSync } from 'node:fs';
6import { execFileSync } from 'node:child_process';
7import { createHash } from 'node:crypto';
8import { GitHttp } from '../../src/client.js';
9import { parsePack, resolvePack } from '../../src/pack.js';
10import { scanPackIndex } from '../../src/packscanindex.js';
11import { resolvePackRange } from '../../src/packrangeresolve.js';
12import { rangeFromBytes } from '../../src/rangesource.js';
13import { pushRepo } from '../../src/repo-node.js';
14import { Tracer } from '../../src/otel.js';
15import { unwrap, unwrapAsync } from '../unwrap.js';
16import { failure, fetchOps } from './ops.mjs';
18const cell = JSON.parse(process.argv[2]);
19// the cell's limits as written: the keys it sets, every other key the default (parseLimits, where the client is made)
20const opts = { tracer: new Tracer(), limits: cell.limits ?? {} };
22let heapPeak = 0;
23let abPeak = 0;
24const sample = setInterval(() => {
25 const m = process.memoryUsage();
26 if (m.heapUsed > heapPeak) heapPeak = m.heapUsed;
27 if (m.arrayBuffers > abPeak) abPeak = m.arrayBuffers;
28}, 5);
29sample.unref();
31/** @param {Iterable<string>} oids */
32function oidsHash(oids) {
33 const hash = createHash('sha1');
34 for (const oid of [...oids].sort()) hash.update(oid);
35 return hash.digest('hex');
36}
38// KiB. /proc/self/status VmHWM is this image's own high-water mark (reset by exec); elsewhere, ru_maxrss.
39function vmHwmKiB() {
40 try {
41 const m = /VmHWM:\s+(\d+)\s+kB/.exec(readFileSync('/proc/self/status', 'utf8'));
42 if (m) return Number(m[1]);
43 } catch {
44 // not Linux
45 }
46 return process.resourceUsage().maxRSS;
47}
49const TYPE_ID = { commit: 1, tree: 2, blob: 3, tag: 4 };
51const ops = {
52 ...fetchOps(cell, opts),
53 async pack() {
54 const buf = new Uint8Array(readFileSync(cell.pack));
55 const parsed = unwrap(parsePack(buf, { limits: opts.limits }));
56 const resolved = unwrap(resolvePack(parsed, () => null, opts.limits));
57 return { n: resolved.objects.size, maxChain: resolved.stats.maxChain, oidsHash: oidsHash(resolved.objects.keys()) };
58 },
59 // resolvePackRange over the pack's bytes. The index is scanned on the defaults; the cell's limits bound the resolve.
60 async resolveRange() {
61 const buf = new Uint8Array(readFileSync(cell.pack));
62 const index = unwrap(await scanPackIndex(new Blob([buf]).stream()));
63 const resolved = unwrap(await resolvePackRange(index, rangeFromBytes(buf), { limits: opts.limits }));
64 return { n: resolved.count, work: resolved.stats.work };
65 },
66 // A thin fetch: the server deltas against the haves, and the bases come from the local store (`cell.store`, a bare
67 // repo holding the haves) through the external lookup, the way a client with those objects resolves them.
68 async fetchThin() {
69 const external = (/** @type {string} */ oid) => {
70 try {
71 const type = execFileSync('git', ['-C', cell.store, 'cat-file', '-t', oid], { encoding: 'utf8' }).trim();
72 const data = execFileSync('git', ['-C', cell.store, 'cat-file', type, oid], { maxBuffer: 1 << 28 });
73 return { type: TYPE_ID[type], data: new Uint8Array(data) };
74 } catch {
75 return null;
76 }
77 };
78 const c = unwrap(GitHttp.create(cell.url, opts));
79 const r = await unwrapAsync(c.fetchPack({ wants: cell.wants, haves: cell.haves, thinPack: true }, { external }));
80 const objects = r.resolved?.objects ?? r.objects ?? new Map();
81 return { n: objects.size, refDelta: r.parsed?.stats.refDelta ?? 0, oidsHash: oidsHash(objects.keys()) };
82 },
83 async pushRepo() {
84 const r = await unwrapAsync(pushRepo(cell.local, cell.url, cell.spec ?? {}, opts));
85 return { updates: r.updates.length, objects: r.objects, thinDeltas: r.thinDeltas, upToDate: r.upToDate };
86 },
87};
89const t0 = performance.now();
90const cpu0 = process.cpuUsage();
91/** @type {Record<string, unknown>} */
92let res;
93try {
94 const op = ops[cell.op];
95 if (!op) throw new Error(`unknown op ${cell.op}`);
96 res = { status: 'ok', ...(await op()) };
97} catch (e) {
98 res = failure(e);
99} finally {
100 clearInterval(sample);
101}
102const cpu = process.cpuUsage(cpu0);
103res.ms = Math.round(performance.now() - t0);
104res.cpuMs = Math.round((cpu.user + cpu.system) / 1000);
105// The peak the kernel saw, including ArrayBuffers the heap flag does not cap. VmHWM, not ru_maxrss: ru_maxrss survives
106// exec, so a cell spawned by a driver that had grown to 500 MB building fixtures reported 500 MB whatever it used.
107res.rssPeakMB = Math.round(vmHwmKiB() / 1024);
108res.heapPeakMB = Math.round(heapPeak / (1024 * 1024));
109res.abPeakMB = Math.round(abPeak / (1024 * 1024));
110process.stdout.write(`${JSON.stringify(res)}\n`);
111process.exit(0);
Diagnostics7 git calls · 248 ms
CallStatusSizeTime0248 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB70 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=69 git.response.bytes=11990 git.response.cache=EXPIRED git.response.duration_ms=70 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit7 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=7 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=7 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetchtree:0200381 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=381 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=294ee88709910bdf98d58eeb6898cd838de94a3a git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=381 git.pkt.payload.max=354 git.sideband.band1.bytes=354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=354 git.pack.trailer=2c7127c2c53a7ca2957296d420246ab2358e3775Open pack map
fetch2002.1 KiB163 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=163 git.response.bytes=2181 git.response.cache=MISS git.response.duration_ms=163 git.fetch.wants=6c2a5f290431326299bef6b62871acee7203f3b9 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2181 git.pkt.payload.max=2154 git.sideband.band1.bytes=2154 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e1c798ce4dc6f8
- Request
- f93c8070…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format