GitHubivarvong

ply

main
ply7 git calls101 ms25 KiB4 cachedprotocol v2
98 lines4.3 KiBRaw
1// One memory cell for resolvePackRange (#248), run as its own process with --expose-gc (test/resolve-memory.test.js).
2// Writes a pack of CHAINS chains to a temp file: each a whole random blob of SIZE bytes and DEPTH OFS deltas on it, each
3// delta keeping all but the last 16 bytes of its base. Scans it from the file, then resolves it over a file-backed
4// RangeSource with a CACHE-byte cache, sampling live arrayBuffers (after gc) at every range read. Prints one JSON line.
5// node --expose-gc test/stream/resolve-cell.mjs CHAINS DEPTH SIZE CACHE
6import { mkdtempSync, rmSync, openSync, writeSync, closeSync, createReadStream } from 'node:fs';
7import { open } from 'node:fs/promises';
8import { join } from 'node:path';
9import { tmpdir } from 'node:os';
10import { deflateSync } from 'node:zlib';
11import { createHash, randomFillSync } from 'node:crypto';
12import { scanPackIndex } from '../../src/packscanindex.js';
13import { resolvePackRange } from '../../src/packrangeresolve.js';
14import { withLimits } from '../../src/limits.js';
16const [chains, depth, size, cache] = process.argv.slice(2).map(Number);
17const TAIL = 16;
18const PIECE = 65_536;
20function objHeader(type, n) {
21 const out = [(type << 4) | (n & 0x0f)];
22 n = Math.floor(n / 16);
23 while (n > 0) { out[out.length - 1] |= 0x80; out.push(n & 0x7f); n = Math.floor(n / 128); }
24 return Uint8Array.from(out);
26function ofs(off) {
27 const out = [off & 0x7f];
28 while ((off = Math.floor(off / 128)) > 0) out.unshift(0x80 | (--off & 0x7f));
29 return Uint8Array.from(out);
31const varint = n => { const out = []; do { let b = n & 0x7f; n = Math.floor(n / 128); if (n) b |= 0x80; out.push(b); } while (n); return out; };
32// copy [0, size - TAIL) of the base, then insert TAIL new bytes
33function delta(tail) {
34 const keep = size - TAIL;
35 return Uint8Array.from([...varint(size), ...varint(size), 0x80 | 0x10 | 0x20 | 0x40, keep & 0xff, (keep >> 8) & 0xff, (keep >> 16) & 0xff, TAIL, ...tail]);
38const dir = mkdtempSync(join(tmpdir(), 'ply-resolve-cell-'));
39const file = join(dir, 'p.pack');
40try {
41 const fd = openSync(file, 'w');
42 const sha = createHash('sha1');
43 let at = 0;
44 const put = bytes => { writeSync(fd, bytes); sha.update(bytes); at += bytes.length; };
45 const head = Buffer.alloc(12);
46 head.write('PACK', 0, 'latin1');
47 head.writeUInt32BE(2, 4);
48 head.writeUInt32BE(chains * (depth + 1), 8);
49 put(head);
50 for (let c = 0; c < chains; c++) {
51 const blob = randomFillSync(new Uint8Array(size));
52 let prev = at;
53 put(objHeader(3, size));
54 put(deflateSync(blob, { level: 1 }));
55 for (let d = 0; d < depth; d++) {
56 const here = at;
57 const program = delta(randomFillSync(new Uint8Array(TAIL)));
58 put(objHeader(6, program.length));
59 put(ofs(here - prev));
60 put(deflateSync(program));
61 prev = here;
62 }
63 }
64 writeSync(fd, sha.digest());
65 closeSync(fd);
66 const packBytes = at + 20;
68 const idx = await scanPackIndex(createReadStream(file, { highWaterMark: PIECE }), { limits: withLimits({ maxObjectSize: size }) });
69 if (!idx.ok) throw new Error(idx.error.message);
70 const fh = await open(file, 'r');
71 const ab = () => { global.gc(); global.gc(); return process.memoryUsage().arrayBuffers; };
72 const base = ab();
73 let peak = base;
74 const source = {
75 size: packBytes,
76 read: async (offset, length) => {
77 peak = Math.max(peak, ab());
78 let o = offset;
79 const end = offset + length;
80 return { ok: true, value: new ReadableStream({
81 async pull(ctl) {
82 if (o >= end) { ctl.close(); return; }
83 const buf = new Uint8Array(Math.min(PIECE, end - o));
84 const { bytesRead } = await fh.read(buf, 0, buf.length, o);
85 o += bytesRead;
86 ctl.enqueue(buf.subarray(0, bytesRead));
87 },
88 }) };
89 },
90 };
91 const t0 = performance.now();
92 const res = await resolvePackRange(idx.value, source, { cacheBytes: cache, limits: withLimits({ maxObjectSize: size, maxDeltaResult: size, maxDeltaChain: depth + 1 }) });
93 const ms = performance.now() - t0;
94 peak = Math.max(peak, ab());
95 await fh.close();
96 if (!res.ok) throw new Error(res.error.message);
97 console.log(JSON.stringify({ chains, depth, size, cache, packBytes, entries: res.value.count, abDelta: peak - base, ms: Math.round(ms), stats: res.value.stats }));
98} finally { rmSync(dir, { recursive: true, force: true }); }
Diagnostics7 git calls · 101 ms
CallStatusSizeTime0101 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=5724
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5724
git.pkt.payload.max=5697
git.sideband.band1.bytes=5697
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=5697
git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9b
Open pack map
fetchtree:0200200 B · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=87
git.response.bytes=200
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=tree:0
git.fetch.wants=c11435e6c34918566bcdf8c5d918e6efddabe286
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=200
git.pkt.payload.max=173
git.sideband.band1.bytes=173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=173
git.pack.trailer=376de41e92a6f5aaa8754d045db726dc04c5ef3a
Open pack map
fetch2001.9 KiB88 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=1930
git.response.cache=MISS
git.response.duration_ms=88
git.fetch.wants=69adaf3eb1d416817752b41305a94de87e6c3d61
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1930
git.pkt.payload.max=1903
git.sideband.band1.bytes=1903
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e2d2836968cf3a
Request
1c548701…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format