GitHubivarvong
ply
main
1// One memory cell for resolvePackRange on a forked delta family larger than the cache, run as its own process with
2// --expose-gc (test/resolve-memory.test.js). Writes one pack to a temp file: a whole random blob of SIZE bytes and a
3// spine DEPTH deep on it; at each level two OFS deltas on the same base (the second is the next spine) and one leaf on
4// the first: 3 x DEPTH + 1 objects of SIZE bytes, each its base with the last 16 bytes replaced. Resolves it over a
5// file-backed RangeSource with a CACHE-byte cache, sampling live arrayBuffers (after gc) at every range read and at
6// every piece pulled, and checks every oid against node:crypto over the bytes the edits describe. One JSON line.
7// node --expose-gc test/stream/resolve-fork-cell.mjs DEPTH SIZE CACHE
8import { mkdtempSync, rmSync, openSync, writeSync, closeSync, createReadStream } from 'node:fs';
9import { open } from 'node:fs/promises';
10import { join } from 'node:path';
11import { tmpdir } from 'node:os';
12import { deflateSync } from 'node:zlib';
13import { createHash, randomFillSync } from 'node:crypto';
14import { scanPackIndex } from '../../src/packscanindex.js';
15import { resolvePackRange } from '../../src/packrangeresolve.js';
16import { withLimits } from '../../src/limits.js';
18const [depth, size, cache] = process.argv.slice(2).map(Number);
19const TAIL = 16;
20const PIECE = 65_536;
22function objHeader(type, n) {
23 const out = [(type << 4) | (n & 0x0f)];
24 n = Math.floor(n / 16);
25 while (n > 0) { out[out.length - 1] |= 0x80; out.push(n & 0x7f); n = Math.floor(n / 128); }
26 return Uint8Array.from(out);
27}
28function ofs(off) {
29 const out = [off & 0x7f];
30 while ((off = Math.floor(off / 128)) > 0) out.unshift(0x80 | (--off & 0x7f));
31 return Uint8Array.from(out);
32}
33const varint = n => { const out = []; do { let b = n & 0x7f; n = Math.floor(n / 128); if (n) b |= 0x80; out.push(b); } while (n); return out; };
34// copy [0, size - TAIL) of the base, then insert TAIL new bytes
35function delta(tail) {
36 const keep = size - TAIL;
37 return Uint8Array.from([...varint(size), ...varint(size), 0x80 | 0x10 | 0x20 | 0x40, keep & 0xff, (keep >> 8) & 0xff, (keep >> 16) & 0xff, TAIL, ...tail]);
38}
40const dir = mkdtempSync(join(tmpdir(), 'ply-resolve-fork-cell-'));
41const file = join(dir, 'p.pack');
42try {
43 const fd = openSync(file, 'w');
44 const sha = createHash('sha1');
45 let at = 0;
46 const put = bytes => { writeSync(fd, bytes); sha.update(bytes); at += bytes.length; };
47 const entries = 3 * depth + 1;
48 const head = Buffer.alloc(12);
49 head.write('PACK', 0, 'latin1');
50 head.writeUInt32BE(2, 4);
51 head.writeUInt32BE(entries, 8);
52 put(head);
53 const want = [];
54 {
55 // every object is the root's first size - TAIL bytes and its own tail, so its oid needs no copy of it
56 const root = randomFillSync(new Uint8Array(size));
57 const oidOf = tail => createHash('sha1').update(`blob ${size}\0`).update(root.subarray(0, size - TAIL)).update(tail).digest('hex');
58 want.push(oidOf(root.subarray(size - TAIL)));
59 const edit = baseAt => {
60 const here = at;
61 const tail = randomFillSync(new Uint8Array(TAIL));
62 const program = delta(tail);
63 put(objHeader(6, program.length));
64 put(ofs(here - baseAt));
65 put(deflateSync(program));
66 want.push(oidOf(tail));
67 return here;
68 };
69 let spine = at;
70 put(objHeader(3, size));
71 put(deflateSync(root, { level: 1 }));
72 for (let k = 0; k < depth; k++) {
73 const first = edit(spine);
74 const second = edit(spine);
75 edit(first);
76 spine = second;
77 }
78 }
79 writeSync(fd, sha.digest());
80 closeSync(fd);
81 const packBytes = at + 20;
83 const limits = withLimits({ maxObjectSize: size, maxDeltaResult: size, maxDeltaChain: depth + 2 });
84 const idx = await scanPackIndex(createReadStream(file, { highWaterMark: PIECE }), { limits });
85 if (!idx.ok) throw new Error(idx.error.message);
86 const fh = await open(file, 'r');
87 const ab = () => { global.gc(); global.gc(); return process.memoryUsage().arrayBuffers; };
88 const base = ab();
89 let peak = base;
90 let samples = 0;
91 const sample = () => { samples += 1; peak = Math.max(peak, ab()); };
92 const source = {
93 size: packBytes,
94 read: async (offset, length) => {
95 sample();
96 let o = offset;
97 const end = offset + length;
98 return { ok: true, value: new ReadableStream({
99 async pull(ctl) {
100 sample();
101 if (o >= end) { ctl.close(); return; }
102 const buf = new Uint8Array(Math.min(PIECE, end - o));
103 const { bytesRead } = await fh.read(buf, 0, buf.length, o);
104 o += bytesRead;
105 ctl.enqueue(buf.subarray(0, bytesRead));
106 },
107 }) };
108 },
109 };
110 const res = await resolvePackRange(idx.value, source, { cacheBytes: cache, limits });
111 sample();
112 await fh.close();
113 if (!res.ok) throw new Error(`${res.error.code}: ${res.error.message}`);
114 let oidsRight = 0;
115 for (let i = 0; i < want.length; i++) if (res.value.oid(i).value === want[i]) oidsRight += 1;
116 console.log(JSON.stringify({ depth, size, cache, packBytes, entries: res.value.count, wanted: want.length, oidsRight, abDelta: peak - base, samples, stats: res.value.stats }));
117} finally { rmSync(dir, { recursive: true, force: true }); }
Diagnostics7 git calls · 242 ms
CallStatusSizeTime0242 ms
discover200191 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB92 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=92 git.response.bytes=11990 git.response.cache=EXPIRED git.response.duration_ms=92 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetchtree:0200200 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=200 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=c11435e6c34918566bcdf8c5d918e6efddabe286 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=200 git.pkt.payload.max=173 git.sideband.band1.bytes=173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=173 git.pack.trailer=376de41e92a6f5aaa8754d045db726dc04c5ef3aOpen pack map
fetch2002.2 KiB138 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=138 git.response.bytes=2251 git.response.cache=MISS git.response.duration_ms=138 git.fetch.wants=acea87d03c5e2ea59d01f98996e9463826765e72 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2251 git.pkt.payload.max=2224 git.sideband.band1.bytes=2224 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e2d9edcf2614c2
- Request
- e7364b5e…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format