GitHubivarvong
ply
main
1// The server-under-test target for Cloudflare Artifacts: a fresh repository per scenario run, in the load-test
2// namespace only, created and deleted with the `cf` CLI. Opt-in (scripts/probes.mjs --artifacts); CI never loads this.
3//
4// Credentials: the CLI's own login is used for the control plane; each repository's write token is minted from the CLI's
5// stdout and held in this process's memory only (handed to the recording proxy, which adds the Authorization header).
6// It is never printed, written, put in argv or in a child's environment; every string this module returns goes through
7// scrub(). Every token minted, including the one `repos create` mints by itself, is revoked and proven dead (403)
8// before the repository is deleted, and the deletion is checked by listing.
9// HTTP-level probes (test/probes/wire.mjs) must reach the server with no proxy between (a proxy reframes requests), so
10// a handle also offers `wireAuth`: the header added in-process for a raw socket or ply's fetch, and for curl and stock
11// git a 0600 file in a 0700 temp directory (curl `-H @file`, git `-c include.path=<file>`), so the token is still in
12// no argv or environment; the caller removes the file after the one request.
13import { execFile } from 'node:child_process';
14import { randomBytes } from 'node:crypto';
15import { mkdtempSync, writeFileSync } from 'node:fs';
16import { tmpdir } from 'node:os';
17import { join } from 'node:path';
18import { performance } from 'node:perf_hooks';
19import { startRecordingProxy } from './runner.mjs';
21export const NAMESPACE = 'ply-loadgen'; // the only namespace hostile or load traffic may touch
22const secrets = new Set();
23const keep = s => { if (typeof s === 'string' && s.length >= 8) { secrets.add(s); secrets.add(encodeURIComponent(s)); secrets.add(Buffer.from(`x:${s}`).toString('base64')); } return s; };
24export const scrub = text => { let t = String(text); for (const s of secrets) t = t.split(s).join('[redacted]'); return t; };
26function sh(cmd, args, { cwd, env }) {
27 return new Promise(resolve => {
28 const t0 = performance.now();
29 execFile(cmd, args, { cwd, env, maxBuffer: 16 << 20, timeout: 120_000 }, (e, stdout, stderr) => resolve({ code: e ? (typeof e.code === 'number' ? e.code : 1) : 0, stdout: String(stdout), stderr: String(stderr), ms: +(performance.now() - t0).toFixed(0) }));
30 });
31}
32const json = text => { const a = text.indexOf('{'), b = text.indexOf('['); const i = a < 0 ? b : b < 0 ? a : Math.min(a, b); try { return i < 0 ? null : JSON.parse(text.slice(i)); } catch { return null; } };
33const ids = node => { const out = []; const visit = n => { if (Array.isArray(n)) n.forEach(visit); else if (n && typeof n === 'object') { if (typeof n.id === 'string' && n.id) out.push(n.id); Object.values(n).forEach(visit); } }; visit(node); return out; };
34const findStrings = (node, keys, out = []) => {
35 if (Array.isArray(node)) { for (const x of node) findStrings(x, keys, out); return out; }
36 if (node && typeof node === 'object') for (const [k, v] of Object.entries(node)) { if (keys.includes(k) && typeof v === 'string') out.push(v); findStrings(v, keys, out); }
37 return out;
38};
39const sleep = ms => new Promise(r => setTimeout(r, ms));
41// cli: { bin, cwd, env } for the `cf` CLI. log: (record) => void for the control-plane audit (scrubbed by the caller).
42export function artifactsTarget({ cli, runId, log }) {
43 if (!/^[a-z0-9-]+$/.test(runId)) throw new Error('runId must be lowercase alphanumerics and dashes');
44 const cf = async (label, args) => { const r = await sh(cli.bin, args, cli); log({ kind: 'cf', label, args: args.filter(a => !secrets.has(a)), code: r.code, ms: r.ms, stderr: r.code ? scrub(r.stderr.trim().slice(0, 300)) : '' }); return r; };
45 let n = 0;
46 const tokens = []; // { id, label, repo, revoked, probe, ray }
47 const repos = []; // { name, created, deleted, listedAfter }
49 // A revoked token must be refused: GET the upload-pack advertisement with it until 403 (propagation), at most ~12 s.
50 async function proveDead(remote, plain) {
51 let status = 0, ray = null;
52 for (let i = 0; i < 6; i++) {
53 const res = await fetch(`${remote}/info/refs?service=git-upload-pack`, { headers: { authorization: `Bearer ${plain}` }, redirect: 'manual' });
54 status = res.status; ray = res.headers.get('cf-ray'); await res.body?.cancel();
55 log({ kind: 'http', label: 'revoked-token probe', status, cfRay: ray });
56 if (status === 403) break;
57 await sleep(2000);
58 }
59 return { status, ray };
60 }
61 async function revoke(t, remote, plain) {
62 let ok = false;
63 for (let i = 0; i < 4 && !ok; i++) { ok = (await cf('token revoke', ['artifacts', 'namespaces', 'tokens', 'revoke', t.id, '--namespace', NAMESPACE, '--force'])).code === 0; if (!ok) await sleep(2000); }
64 t.revoked = ok;
65 if (plain !== null) { const p = await proveDead(remote, plain); t.probe = p.status; t.ray = p.ray; }
66 }
68 return {
69 tokens, repos,
70 // Create the repository, revoke whatever it minted by itself, mint a write token, and start a proxy that adds it.
71 async open(label) {
72 const name = `ply-probe-${runId}-${String(n++).padStart(2, '0')}`;
73 const rec = { name, label, created: false, deleted: false, listedAfter: null };
74 repos.push(rec);
75 const c = await cf('repo create', ['artifacts', 'namespaces', 'repos', 'create', NAMESPACE, '--name', name, '--default-branch', 'main', '--description', `ply differential probe ${label} (deleted after)`]);
76 const j = c.code === 0 ? json(c.stdout) : null;
77 const autos = findStrings(j, ['token', 'plaintext']).map(keep);
78 const remote = findStrings(j, ['remote']).find(u => /^https:\/\/[a-z0-9]+\.artifacts\.cloudflare\.net\/git\//.test(u)) ?? null;
79 if (remote === null) throw new Error(`repo create gave no remote (exit ${c.code}): ${scrub(c.stderr.trim().slice(0, 300))}`);
80 rec.created = true;
81 const listed = await cf('repo tokens list', ['artifacts', 'namespaces', 'repos', 'tokens', 'list', '--namespace', NAMESPACE, '--name', name, '--state', 'active']);
82 for (const id of listed.code === 0 ? ids(json(listed.stdout)) : []) {
83 const t = { id, label: 'auto (repos create)', repo: name, revoked: false, probe: null, ray: null };
84 tokens.push(t);
85 await revoke(t, remote, autos[0] ?? null);
86 }
87 const m = await cf('token create (write)', ['artifacts', 'namespaces', 'tokens', 'create', NAMESPACE, '--repo', name, '--scope', 'write', '--ttl', '1800']);
88 const mj = m.code === 0 ? json(m.stdout) : null;
89 const plain = typeof mj?.plaintext === 'string' ? keep(mj.plaintext) : null;
90 if (plain === null || typeof mj?.id !== 'string') throw new Error(`token create failed (exit ${m.code})`);
91 const token = { id: mj.id, label: 'write', repo: name, revoked: false, probe: null, ray: null };
92 tokens.push(token);
93 const u = new URL(remote);
94 const proxy = await startRecordingProxy({ upstream: u.origin, authorization: `Bearer ${plain}` });
95 // a bearer token never minted (random, same shape as nothing in particular), the revoked auto token, the write token
96 const bogus = keep(`bogus${randomBytes(20).toString('hex')}`);
97 const tokenFor = kind => (kind === 'write' || kind === 'missing-repo' ? plain : kind === 'bogus' ? bogus : kind === 'revoked' ? (autos[0] ?? null) : null);
98 const secretFile = (kind, text) => {
99 const t = tokenFor(kind);
100 if (t === null) return null;
101 const f = join(mkdtempSync(join(tmpdir(), 'ply-wire-auth-')), kind);
102 writeFileSync(f, text(t), { mode: 0o600 });
103 return f;
104 };
105 const missing = remote.replace(/\/?$/, '').replace(/\.git$/, '') + '-absent.git';
106 const wireAuth = {
107 headers: (kind, h) => { const t = tokenFor(kind); if (t !== null) h.authorization = `Bearer ${t}`; },
108 curlFile: kind => secretFile(kind, t => `Authorization: Bearer ${t}\n`),
109 gitFile: kind => secretFile(kind, t => `[http]\n\textraHeader = Authorization: Bearer ${t}\n`),
110 url: kind => (kind === 'missing-repo' ? missing : remote.replace(/\/$/, '')),
111 hasRevoked: () => autos.length > 0,
112 };
113 return {
114 name, proxy, repoPath: u.pathname.replace(/\/$/, ''), wireAuth,
115 async close() {
116 await proxy.close();
117 await revoke(token, remote, plain);
118 const left = await cf('repo tokens list', ['artifacts', 'namespaces', 'repos', 'tokens', 'list', '--namespace', NAMESPACE, '--name', name, '--state', 'active']);
119 for (const id of left.code === 0 ? ids(json(left.stdout)) : []) {
120 const t = { id, label: 'leftover', repo: name, revoked: false, probe: null, ray: null };
121 tokens.push(t);
122 await revoke(t, remote, null);
123 }
124 for (let i = 0; i < 3 && !rec.deleted; i++) rec.deleted = (await cf('repo delete', ['artifacts', 'namespaces', 'repos', 'delete', name, '--namespace', NAMESPACE, '--force'])).code === 0;
125 const after = await cf('repo list (gone?)', ['artifacts', 'namespaces', 'repos', 'list', '--namespace', NAMESPACE, '--search', name]);
126 rec.listedAfter = after.code === 0 ? JSON.stringify(json(after.stdout) ?? []).includes(`"${name}"`) : null;
127 },
128 };
129 },
130 };
131}
Diagnostics9 git calls · 103 ms
CallStatusSizeTime0103 ms
discover200191 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=11990 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=15 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0 · deepen 1200347 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=347 git.pkt.payload.max=247 git.sideband.band1.bytes=247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=247 git.pack.trailer=4b72f39b5ea57c5ae3009f90b53e3a6a155093e7Open pack map
fetchcombine:blob:none+tree:2 · deepen 1429—15 ms
git.upload-pack fetch
http.response.status_code=429
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.cache=BYPASS
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
error.type=rate_limited
error=HTTP 429 git-upload-pack: \u000d
<html>\u000d
<head>\u000d
<meta content="origin" name="referrer">\u000d
<title>Rate limit · GitHub</title>\u000d
<meta name="vfetchtree:0200919 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02005.6 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=5724 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=58fabb6768b3f737ee9cd8bac647bdc27405fb4d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5724 git.pkt.payload.max=5697 git.sideband.band1.bytes=5697 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=5697 git.pack.trailer=be7e0453974a5fe832f9065a33bb7e3434c10e9bOpen pack map
fetchtree:0200305 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=79 git.response.bytes=305 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=facf357dda5424ce8c1bda8ebca6cacc9b5bfc3a git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=305 git.pkt.payload.max=278 git.sideband.band1.bytes=278 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=278 git.pack.trailer=36c263093b3db739addfdbaa3928c1028217ea9eOpen pack map
fetch2003.6 KiB79 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3659 git.response.cache=MISS git.response.duration_ms=79 git.fetch.wants=97cc232ba67a3407cccb4a691aa1c274dfc10e2c git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3659 git.pkt.payload.max=3632 git.sideband.band1.bytes=3632 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e1c32cb9a636cd
- Request
- b2d6aea8…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format