Artifactspublic-v0

ply

main
ply4 git calls208 ms41 KiB2 cachedprotocol v2
146 lines7.7 KiBRaw
1// Seeds (or deletes) the throwaway Artifacts repo the harness reads: ~1 GiB of history pushed as one commit per push,
2// each push under the 64 MiB request cap, with stock git. The worker then fetches the whole history as one pack.
3//
4// node bench/worker-mem/seed.mjs create + push, write .data/seed.json
5// node bench/worker-mem/seed.mjs --delete delete the repo named in .data/seed.json
6//
7// Content is deterministic (AES-CTR keystream), mostly incompressible so pack bytes ≈ data bytes, plus one commit of
8// near-duplicate text files so the pack also carries real deltas. One 30 MiB blob (under the 32 MiB object cap) is the
9// read cell's target.
10import { spawnSync } from 'node:child_process';
11import { createCipheriv, createHash } from 'node:crypto';
12import { mkdirSync, rmSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
13import { join, dirname } from 'node:path';
14import { NO_MAINTENANCE } from '../../test/server.js';
15import { DATA, NAMESPACE, cfJson, cfRaw, remember, writeSecret, stamp, redact, sleep } from './lib.mjs';
17const MIB = 1024 * 1024;
18const SEED_FILE = join(DATA, 'seed.json');
19const COMMITS = Number(process.env.MEMLAB_COMMITS ?? 18);
20const FILES_PER_COMMIT = 7;
21const FILE_MIB = 8;
22const BIG_BLOB_MIB = 30;
23const TEXT_FILES = 1500;
25const keystream = (label, n) => {
26 const key = createHash('sha256').update(`ply-memlab:${label}`).digest().subarray(0, 16);
27 return createCipheriv('aes-128-ctr', key, Buffer.alloc(16)).update(Buffer.alloc(n));
28};
30function git(cwd, args, opts = {}) {
31 const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: { ...process.env, ...opts.env }, maxBuffer: 64 * MIB });
32 if (r.status !== 0 && !opts.allowFail) throw new Error(`git ${args[0]} failed: ${redact(r.stderr).slice(0, 800)}`);
33 return r;
36function write(src, rel, data) {
37 const p = join(src, rel);
38 mkdirSync(dirname(p), { recursive: true });
39 writeFileSync(p, data);
42function ids(x, out = new Set()) {
43 if (Array.isArray(x)) for (const v of x) ids(v, out);
44 else if (x && typeof x === 'object') {
45 if (typeof x.id === 'string' && ('state' in x || 'expires_at' in x || 'scope' in x)) out.add(x.id);
46 for (const v of Object.values(x)) ids(v, out);
47 }
48 return out;
51export function revokeAll(name) {
52 const listed = cfJson(['artifacts', 'namespaces', 'repos', 'tokens', 'list', '--namespace', NAMESPACE, '--name', name, '--state', 'active']);
53 const all = [...ids(listed)];
54 for (const id of all) cfJson(['artifacts', 'namespaces', 'tokens', 'revoke', id, '--namespace', NAMESPACE, '--force']);
55 return all.length;
58export function mint(name, scope, ttl) {
59 const m = cfJson(['artifacts', 'namespaces', 'tokens', 'create', NAMESPACE, '--repo', name, '--scope', scope, '--ttl', String(ttl)]);
60 return { token: remember(m.plaintext), id: m.id };
63// 401/403 from the advertisement with this token means it no longer works.
64export async function refused(remote, token) {
65 const res = await fetch(`${remote}/info/refs?service=git-upload-pack`, {
66 headers: { authorization: `Basic ${Buffer.from(`x:${token}`).toString('base64')}`, 'git-protocol': 'version=2' },
67 });
68 await res.arrayBuffer();
69 return res.status === 401 || res.status === 403;
72async function seed() {
73 const name = `ply-memlab-seed-${stamp().toLowerCase()}`;
74 const created = cfJson(['artifacts', 'namespaces', 'repos', 'create', NAMESPACE, '--name', name, '--default-branch', 'main', '--description', 'ply bench/worker-mem seed (kept for reruns; delete with seed.mjs --delete)']);
75 const remote = created.remote ?? created.result?.remote;
76 if (!/^https:\/\/[a-z0-9]+\.artifacts\.cloudflare\.net\/git\//.test(remote ?? '')) throw new Error('repo create returned no Artifacts remote');
77 writeFileSync(SEED_FILE, JSON.stringify({ name, remote, state: 'created' }, null, 1));
78 console.log(`created ${NAMESPACE}/${name}; revoked ${revokeAll(name)} auto token(s)`);
79 const write_ = mint(name, 'write', 7200);
80 const auth = join(DATA, 'auth.gitconfig');
81 writeSecret(auth, `[http]\n\textraHeader = Authorization: Basic ${Buffer.from(`x:${write_.token}`).toString('base64')}\n`);
82 const src = join(DATA, 'src');
83 rmSync(src, { recursive: true, force: true });
84 mkdirSync(src, { recursive: true });
85 git(src, ['init', '-q', '-b', 'main']);
86 for (const [k, v] of [['user.name', 'ply memlab'], ['user.email', 'memlab@example.invalid'], ...NO_MAINTENANCE, ['core.compression', '6']]) git(src, ['config', k, v]);
87 const date = { GIT_AUTHOR_DATE: '1700000000 +0000', GIT_COMMITTER_DATE: '1700000000 +0000' };
88 const pushes = [];
89 const push = async (sha, label) => {
90 for (let attempt = 1; attempt <= 4; attempt++) {
91 const t0 = Date.now();
92 const r = git(src, ['-c', `include.path=${auth}`, 'push', '--porcelain', remote, `${sha}:refs/heads/main`], { allowFail: true });
93 if (r.status === 0) { pushes.push({ label, sha, ms: Date.now() - t0, attempt }); console.log(`pushed ${label} ${sha.slice(0, 12)} in ${Date.now() - t0} ms`); return; }
94 console.log(`push ${label} attempt ${attempt} failed: ${redact(r.stderr).slice(0, 300)}`);
95 await sleep(5000 * attempt);
96 }
97 throw new Error(`push ${label} failed 4 times`);
98 };
99 try {
100 // commit 0: the read cell's 30 MiB blob and near-duplicate text (real deltas)
101 write(src, 'big/blob30.bin', keystream('big', BIG_BLOB_MIB * MIB));
102 const base = keystream('text-base', 24 * 1024).toString('hex');
103 for (let f = 0; f < TEXT_FILES; f++) {
104 const noise = keystream(`text-${f}`, 64).toString('hex');
105 const at = (f * 977) % (base.length - 200);
106 write(src, `text/d${String(f % 30).padStart(2, '0')}/f${f}.txt`, `${base.slice(0, at)}${noise}${base.slice(at + noise.length)}\n`);
107 }
108 git(src, ['add', '-A']);
109 git(src, ['commit', '-q', '-m', 'memlab seed 0: big blob + text'], { env: date });
110 const blob = git(src, ['rev-parse', 'HEAD:big/blob30.bin']).stdout.trim();
111 await push(git(src, ['rev-parse', 'HEAD']).stdout.trim(), 'c0');
112 for (let c = 1; c <= COMMITS; c++) {
113 for (let f = 0; f < FILES_PER_COMMIT; f++) write(src, `bin/c${String(c).padStart(2, '0')}/f${f}.bin`, keystream(`bin-${c}-${f}`, FILE_MIB * MIB));
114 git(src, ['add', '-A']);
115 git(src, ['commit', '-q', '-m', `memlab seed ${c}`], { env: date });
116 await push(git(src, ['rev-parse', 'HEAD']).stdout.trim(), `c${c}`);
117 }
118 const head = git(src, ['rev-parse', 'HEAD']).stdout.trim();
119 const objects = git(src, ['rev-list', '--objects', '--all']).stdout.trim().split('\n').length;
120 const packed = spawnSync('git', ['pack-objects', '--stdout', '--revs', '--delta-base-offset'], { cwd: src, input: `${head}\n`, maxBuffer: 4096 * MIB });
121 const meta = { name, remote, state: 'seeded', head, blob, blobBytes: BIG_BLOB_MIB * MIB, commits: COMMITS + 1, objects, localPackBytes: packed.stdout.length, pushes, seededAt: new Date().toISOString() };
122 writeFileSync(SEED_FILE, JSON.stringify(meta, null, 1));
123 console.log(`seeded head ${head} objects ${objects} local pack ${packed.stdout.length} bytes`);
124 } finally {
125 rmSync(auth, { force: true });
126 revokeAll(name);
127 console.log(`write token revoked; refused afterwards: ${await refused(remote, write_.token)}`);
128 rmSync(src, { recursive: true, force: true });
129 }
132function remove() {
133 if (!existsSync(SEED_FILE)) throw new Error('no .data/seed.json');
134 const { name } = JSON.parse(readFileSync(SEED_FILE, 'utf8'));
135 revokeAll(name);
136 const r = cfRaw(['artifacts', 'namespaces', 'repos', 'delete', name, '--namespace', NAMESPACE, '--force']);
137 if (r.code !== 0) throw new Error(`delete failed: ${r.stderr.slice(0, 300)}`);
138 rmSync(SEED_FILE);
139 console.log(`deleted ${NAMESPACE}/${name}`);
142if (import.meta.url === `file://${process.argv[1]}`) {
143 mkdirSync(DATA, { recursive: true });
144 if (process.argv.includes('--delete')) remove();
145 else await seed();
Diagnostics4 git calls · 208 ms
CallStatusSizeTime0208 ms
discover200152 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=203
git.response.bytes=38434
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=38434
git.pkt.payload.max=38339
git.sideband.band1.bytes=38338
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=63
git.pack.bytes=38338
git.pack.ofs_delta=0
git.pack.ref_delta=8
git.pack.chain.max=8
git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77
git.pack.full=55
Open pack map
fetch2003.2 KiB203 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3230
git.response.cache=MISS
git.response.duration_ms=203
git.fetch.wants=62c708eb91f1eb9e0c98f16ce15753fe78f56f0a
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3230
git.pkt.payload.max=3209
git.sideband.band1.bytes=3208
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e36296ed77b23b
Request
48edee38…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format