GitHubivarvong

ply

main
ply7 git calls206 ms21 KiB4 cachedprotocol v2
146 lines7.7 KiBRaw
1// Seeds (or deletes) the throwaway Artifacts repo the harness reads: ~1 GiB of history pushed as one commit per push,
2// each push under the 64 MiB request cap, with stock git. The worker then fetches the whole history as one pack.
3//
4// node bench/worker-mem/seed.mjs create + push, write .data/seed.json
5// node bench/worker-mem/seed.mjs --delete delete the repo named in .data/seed.json
6//
7// Content is deterministic (AES-CTR keystream), mostly incompressible so pack bytes ≈ data bytes, plus one commit of
8// near-duplicate text files so the pack also carries real deltas. One 30 MiB blob (under the 32 MiB object cap) is the
9// read cell's target.
10import { spawnSync } from 'node:child_process';
11import { createCipheriv, createHash } from 'node:crypto';
12import { mkdirSync, rmSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
13import { join, dirname } from 'node:path';
14import { NO_MAINTENANCE } from '../../test/server.js';
15import { DATA, NAMESPACE, cfJson, cfRaw, remember, writeSecret, stamp, redact, sleep } from './lib.mjs';
17const MIB = 1024 * 1024;
18const SEED_FILE = join(DATA, 'seed.json');
19const COMMITS = Number(process.env.MEMLAB_COMMITS ?? 18);
20const FILES_PER_COMMIT = 7;
21const FILE_MIB = 8;
22const BIG_BLOB_MIB = 30;
23const TEXT_FILES = 1500;
25const keystream = (label, n) => {
26 const key = createHash('sha256').update(`ply-memlab:${label}`).digest().subarray(0, 16);
27 return createCipheriv('aes-128-ctr', key, Buffer.alloc(16)).update(Buffer.alloc(n));
28};
30function git(cwd, args, opts = {}) {
31 const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: { ...process.env, ...opts.env }, maxBuffer: 64 * MIB });
32 if (r.status !== 0 && !opts.allowFail) throw new Error(`git ${args[0]} failed: ${redact(r.stderr).slice(0, 800)}`);
33 return r;
36function write(src, rel, data) {
37 const p = join(src, rel);
38 mkdirSync(dirname(p), { recursive: true });
39 writeFileSync(p, data);
42function ids(x, out = new Set()) {
43 if (Array.isArray(x)) for (const v of x) ids(v, out);
44 else if (x && typeof x === 'object') {
45 if (typeof x.id === 'string' && ('state' in x || 'expires_at' in x || 'scope' in x)) out.add(x.id);
46 for (const v of Object.values(x)) ids(v, out);
47 }
48 return out;
51export function revokeAll(name) {
52 const listed = cfJson(['artifacts', 'namespaces', 'repos', 'tokens', 'list', '--namespace', NAMESPACE, '--name', name, '--state', 'active']);
53 const all = [...ids(listed)];
54 for (const id of all) cfJson(['artifacts', 'namespaces', 'tokens', 'revoke', id, '--namespace', NAMESPACE, '--force']);
55 return all.length;
58export function mint(name, scope, ttl) {
59 const m = cfJson(['artifacts', 'namespaces', 'tokens', 'create', NAMESPACE, '--repo', name, '--scope', scope, '--ttl', String(ttl)]);
60 return { token: remember(m.plaintext), id: m.id };
63// 401/403 from the advertisement with this token means it no longer works.
64export async function refused(remote, token) {
65 const res = await fetch(`${remote}/info/refs?service=git-upload-pack`, {
66 headers: { authorization: `Basic ${Buffer.from(`x:${token}`).toString('base64')}`, 'git-protocol': 'version=2' },
67 });
68 await res.arrayBuffer();
69 return res.status === 401 || res.status === 403;
72async function seed() {
73 const name = `ply-memlab-seed-${stamp().toLowerCase()}`;
74 const created = cfJson(['artifacts', 'namespaces', 'repos', 'create', NAMESPACE, '--name', name, '--default-branch', 'main', '--description', 'ply bench/worker-mem seed (kept for reruns; delete with seed.mjs --delete)']);
75 const remote = created.remote ?? created.result?.remote;
76 if (!/^https:\/\/[a-z0-9]+\.artifacts\.cloudflare\.net\/git\//.test(remote ?? '')) throw new Error('repo create returned no Artifacts remote');
77 writeFileSync(SEED_FILE, JSON.stringify({ name, remote, state: 'created' }, null, 1));
78 console.log(`created ${NAMESPACE}/${name}; revoked ${revokeAll(name)} auto token(s)`);
79 const write_ = mint(name, 'write', 7200);
80 const auth = join(DATA, 'auth.gitconfig');
81 writeSecret(auth, `[http]\n\textraHeader = Authorization: Basic ${Buffer.from(`x:${write_.token}`).toString('base64')}\n`);
82 const src = join(DATA, 'src');
83 rmSync(src, { recursive: true, force: true });
84 mkdirSync(src, { recursive: true });
85 git(src, ['init', '-q', '-b', 'main']);
86 for (const [k, v] of [['user.name', 'ply memlab'], ['user.email', 'memlab@example.invalid'], ...NO_MAINTENANCE, ['core.compression', '6']]) git(src, ['config', k, v]);
87 const date = { GIT_AUTHOR_DATE: '1700000000 +0000', GIT_COMMITTER_DATE: '1700000000 +0000' };
88 const pushes = [];
89 const push = async (sha, label) => {
90 for (let attempt = 1; attempt <= 4; attempt++) {
91 const t0 = Date.now();
92 const r = git(src, ['-c', `include.path=${auth}`, 'push', '--porcelain', remote, `${sha}:refs/heads/main`], { allowFail: true });
93 if (r.status === 0) { pushes.push({ label, sha, ms: Date.now() - t0, attempt }); console.log(`pushed ${label} ${sha.slice(0, 12)} in ${Date.now() - t0} ms`); return; }
94 console.log(`push ${label} attempt ${attempt} failed: ${redact(r.stderr).slice(0, 300)}`);
95 await sleep(5000 * attempt);
96 }
97 throw new Error(`push ${label} failed 4 times`);
98 };
99 try {
100 // commit 0: the read cell's 30 MiB blob and near-duplicate text (real deltas)
101 write(src, 'big/blob30.bin', keystream('big', BIG_BLOB_MIB * MIB));
102 const base = keystream('text-base', 24 * 1024).toString('hex');
103 for (let f = 0; f < TEXT_FILES; f++) {
104 const noise = keystream(`text-${f}`, 64).toString('hex');
105 const at = (f * 977) % (base.length - 200);
106 write(src, `text/d${String(f % 30).padStart(2, '0')}/f${f}.txt`, `${base.slice(0, at)}${noise}${base.slice(at + noise.length)}\n`);
107 }
108 git(src, ['add', '-A']);
109 git(src, ['commit', '-q', '-m', 'memlab seed 0: big blob + text'], { env: date });
110 const blob = git(src, ['rev-parse', 'HEAD:big/blob30.bin']).stdout.trim();
111 await push(git(src, ['rev-parse', 'HEAD']).stdout.trim(), 'c0');
112 for (let c = 1; c <= COMMITS; c++) {
113 for (let f = 0; f < FILES_PER_COMMIT; f++) write(src, `bin/c${String(c).padStart(2, '0')}/f${f}.bin`, keystream(`bin-${c}-${f}`, FILE_MIB * MIB));
114 git(src, ['add', '-A']);
115 git(src, ['commit', '-q', '-m', `memlab seed ${c}`], { env: date });
116 await push(git(src, ['rev-parse', 'HEAD']).stdout.trim(), `c${c}`);
117 }
118 const head = git(src, ['rev-parse', 'HEAD']).stdout.trim();
119 const objects = git(src, ['rev-list', '--objects', '--all']).stdout.trim().split('\n').length;
120 const packed = spawnSync('git', ['pack-objects', '--stdout', '--revs', '--delta-base-offset'], { cwd: src, input: `${head}\n`, maxBuffer: 4096 * MIB });
121 const meta = { name, remote, state: 'seeded', head, blob, blobBytes: BIG_BLOB_MIB * MIB, commits: COMMITS + 1, objects, localPackBytes: packed.stdout.length, pushes, seededAt: new Date().toISOString() };
122 writeFileSync(SEED_FILE, JSON.stringify(meta, null, 1));
123 console.log(`seeded head ${head} objects ${objects} local pack ${packed.stdout.length} bytes`);
124 } finally {
125 rmSync(auth, { force: true });
126 revokeAll(name);
127 console.log(`write token revoked; refused afterwards: ${await refused(remote, write_.token)}`);
128 rmSync(src, { recursive: true, force: true });
129 }
132function remove() {
133 if (!existsSync(SEED_FILE)) throw new Error('no .data/seed.json');
134 const { name } = JSON.parse(readFileSync(SEED_FILE, 'utf8'));
135 revokeAll(name);
136 const r = cfRaw(['artifacts', 'namespaces', 'repos', 'delete', name, '--namespace', NAMESPACE, '--force']);
137 if (r.code !== 0) throw new Error(`delete failed: ${r.stderr.slice(0, 300)}`);
138 rmSync(SEED_FILE);
139 console.log(`deleted ${NAMESPACE}/${name}`);
142if (import.meta.url === `file://${process.argv[1]}`) {
143 mkdirSync(DATA, { recursive: true });
144 if (process.argv.includes('--delete')) remove();
145 else await seed();
Diagnostics7 git calls · 206 ms
CallStatusSizeTime0206 ms
discover200191 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:0200106 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=106
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=72bbbdb37dc951e0de5615d1376ca1b23588856c
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=106
git.pkt.payload.max=79
git.sideband.band1.bytes=79
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=79
git.pack.trailer=1cca45a06ebd0c08bccef52d1cdb92cae1868075
Open pack map
fetchtree:0200291 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=192
git.response.bytes=291
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=d3ab99f804f647a2fe02e623e69e5ccc7a17b935
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=291
git.pkt.payload.max=264
git.sideband.band1.bytes=264
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=264
git.pack.trailer=1fec5d654a53d2752ea2b9c5bd06569c4565d5b6
Open pack map
fetch2003.2 KiB193 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3229
git.response.cache=MISS
git.response.duration_ms=193
git.fetch.wants=62c708eb91f1eb9e0c98f16ce15753fe78f56f0a
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3229
git.pkt.payload.max=3202
git.sideband.band1.bytes=3202
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e318bc7b063e6e
Request
1bd73ab8…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format