main
1// The memory bound of the pull codecs and of fetchSingle, fetchWide, fetchPack and inflate() on them (issue #172), measured: each cell is its own process
2// (test/stream/cell.mjs, MEMORY_CELL_FLAGS) under a 64 MiB and a 128 MiB V8 heap, its input generated as it is read, and the
3// peak of live arrayBuffers over the cell (a gc at every sample, array buffers swept with it) compared with streambound.ts.
4// Every scan cell asserts the bound itself, on node 22 and 24 alike (#249: the per-PR node 22 allowances were the cells'
5// generator sources and V8's background sweep, not ply; see test/memcell.js and test/stream/pieces.mjs). The parent does not import
6// src/, so a mutation run does not re-run the cells for every covered line.
7import { test } from 'vitest';
8import assert from 'node:assert/strict';
9import { spawnSync } from 'node:child_process';
10import { MEMORY_CELL_FLAGS } from './memcell.js';
12const MIB = 1024 * 1024;
13const HOLD = 32_768 + 16_384 + 65_520; // INFLATE_WINDOW + INFLATE_CHUNK + PKT_MAX_SIZE (format.ts; peakBound 'inflate')
14const PKT = 65_520;
15function cell(mode, size, heap) {
16 const r = spawnSync(process.execPath, [...MEMORY_CELL_FLAGS, `--max-old-space-size=${heap}`, '--import', './test/ts-resolve.mjs', 'test/stream/cell.mjs', mode, String(size)], { encoding: 'utf8', env: process.env, maxBuffer: 1 << 20 });
17 assert.equal(r.status, 0, `${mode} ${size} heap ${heap}: exit ${r.status}\n${r.stderr}`);
18 return JSON.parse(r.stdout.trim().split('\n').at(-1));
19}
21for (const heap of [64, 128]) {
22 test(`heap ${heap} MiB: a 64 MiB member inflated chunk by chunk holds the window and a chunk, not the object`, () => {
23 const r = cell('inflate-drop', 64 * MIB, heap);
24 console.log(JSON.stringify(r));
25 assert.equal(r.bound, HOLD);
26 assert.ok(r.abDelta <= r.bound, `inflate held ${r.abDelta} > ${r.bound}`);
27 });
28 test(`heap ${heap} MiB: a pack of 100,000 one-byte blobs scanned object by object holds one inflate record`, () => {
29 const r = cell('scan-drop', 100_000, heap);
30 console.log(JSON.stringify(r));
31 assert.ok(r.abDelta <= HOLD, `scan held ${r.abDelta} > ${HOLD}`);
32 });
33 test(`heap ${heap} MiB: a pack of three 96 MiB blobs, past every default cap, scanned under WORK_BUDGET holds one inflate record`, () => {
34 const r = cell('scan-work', 96 * MIB, heap);
35 console.log(JSON.stringify(r));
36 assert.equal(r.inflated, 3 * 96 * MIB);
37 // 84,284 B on node 22 and 24 (#249). With generator sources node 22 read 133,436 B: three stale 16 KiB pieces in
38 // the suspended frames of the cell's own generators.
39 assert.ok(r.abDelta <= HOLD, `scan held ${r.abDelta} > ${HOLD}`);
40 });
41 test(`heap ${heap} MiB: scanPackIndex of 200,000 one-byte blobs (oids on) holds the scan and 51 bytes per entry`, () => {
42 const r = cell('scan-index', 200_000, heap);
43 console.log(JSON.stringify(r));
44 // 31 bytes of columns and a 20-byte oid per entry, sized to the header's count
45 assert.equal(r.perEntry, r.entryBytes + 20);
46 assert.equal(r.heldBytes, 200_000 * (r.entryBytes + 20));
47 // the last doubling holds the old columns and the new ones at once; the chunk being fed is inside HOLD's unread input
48 assert.ok(r.abDelta <= HOLD + 2 * r.heldBytes, `index held ${r.abDelta} > ${HOLD} + 2 x ${r.heldBytes}`);
49 });
50 test(`heap ${heap} MiB: scanPackIndex of three 96 MiB blobs, past every default cap, holds one inflate record`, () => {
51 const r = cell('scan-index-big', 96 * MIB, heap);
52 console.log(JSON.stringify(r));
53 assert.ok(r.packBytes > 3 * 96 * MIB);
54 // 84,284 B on node 24; node 22 reads 84,289 B or 100,673 B. The extra 16 KiB there is one inflate chunk that a stale
55 // register of scanPackIndex's own async frame keeps (a heap snapshot at the peak, #249): ply's, and inside the bound.
56 // The 166,204 B node 22 read before #249 was the cell's async-generator source.
57 assert.ok(r.abDelta <= HOLD + r.heldBytes, `index held ${r.abDelta} > ${HOLD} + ${r.heldBytes}`);
58 });
59 test(`heap ${heap} MiB: the async-iterable adapters (#241) hold their decoder's bound: scanPack over 3 x 96 MiB, inflateChunks of 64 MiB, pktFrames of 2,000 frames`, () => {
60 const scan = cell('scan-iter', 96 * MIB, heap);
61 console.log(JSON.stringify(scan));
62 assert.equal(scan.inflated, 3 * 96 * MIB);
63 assert.ok(scan.abDelta <= HOLD, `scanPack held ${scan.abDelta} > ${HOLD}`);
64 const inf = cell('inflate-iter', 64 * MIB, heap);
65 console.log(JSON.stringify(inf));
66 assert.equal(inf.bound, HOLD);
67 assert.ok(inf.abDelta <= HOLD, `inflateChunks held ${inf.abDelta} > ${HOLD}`);
68 const frames = cell('pkt-iter', 2000, heap);
69 console.log(JSON.stringify(frames));
70 assert.equal(frames.bytes, 2000 * 65_516);
71 assert.ok(frames.abDelta <= frames.bound, `pktFrames held ${frames.abDelta} > ${frames.bound}`);
72 });
73 test(`heap ${heap} MiB: inflate() of a 64 MiB member in hand holds the output and the pull's hold, nothing more`, () => {
74 for (const mode of ['inflate-whole', 'inflate-capped']) {
75 const r = cell(mode, 64 * MIB, heap);
76 console.log(JSON.stringify(r));
77 assert.ok(r.abDelta >= 64 * MIB, `${mode}: the output is held`);
78 assert.ok(r.abDelta <= 64 * MIB + HOLD, `${mode}: inflate() held ${r.abDelta} > ${64 * MIB} + ${HOLD}`);
79 }
80 });
81 test(`heap ${heap} MiB: a wide fetch (fetchWide, fetchPack) that retains R bytes of a pack of 2,000 trees holds R and the inflate hold, not the pack`, () => {
82 const drain = cell('fetch-wide-drain', 2000, heap);
83 for (const mode of ['fetch-wide', 'fetch-wide-blobs', 'fetch-pack']) {
84 const r = cell(mode, 2000, heap);
85 console.log(JSON.stringify({ ...r, drain: drain.abDelta }));
86 // what reading the response costs is the transport's (fetch-wide-drain); the fetch adds what it retains (the trees:
87 // the blobs of fetch-wide-blobs are dropped as their headers arrive) and the scan's inflate hold
88 assert.ok(r.abDelta >= r.retained, `${mode}: the trees are held`);
89 assert.ok(r.abDelta - drain.abDelta <= r.bound, `${mode}: added ${r.abDelta - drain.abDelta} > R ${r.retained} + ${HOLD}`);
90 }
91 });
92 test(`heap ${heap} MiB: fetchObject of a 48 MiB blob holds the blob and the inflate hold; before #172 it held the member twice more`, () => {
93 const size = 48 * MIB;
94 const drain = cell('fetch-drain', size, heap);
95 const streamed = cell('fetch-object', size, heap);
96 const buffered = cell('fetch-buffered', size, heap);
97 console.log(JSON.stringify({ drain, streamed, buffered }));
98 // what reading the response costs (the generator's frames, the body chunk, the copied pkt payload) is the
99 // transport's; fetchSingle adds the object, the inflate hold, and spStep's join of the trailer lag and one frame
100 const added = streamed.abDelta - drain.abDelta;
101 assert.ok(added <= size + HOLD + PKT, `fetchSingle added ${added} > ${size} + ${HOLD} + ${PKT}`);
102 assert.ok(added >= size, 'the blob itself is held');
103 assert.ok(buffered.abDelta >= 3 * size, `the buffered read held ${buffered.abDelta}: the member, its join, and the object`);
104 assert.ok(streamed.rss < buffered.rss, `rss ${streamed.rss} vs buffered ${buffered.rss}`);
105 });
106}
Diagnostics4 git calls · 590 ms
CallStatusSizeTime0590 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=584 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2002.5 KiB584 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=2518 git.response.cache=MISS git.response.duration_ms=584 git.fetch.wants=2a279c2df33f3c568656a3df2e8178da0db96a89 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2518 git.pkt.payload.max=2497 git.sideband.band1.bytes=2496 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e280bfce2f0555
- Request
- 259e2135…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format