main
ply4 git calls29 ms46 KiB3 cachedprotocol v2
151 lines8.7 KiBRaw
1#!/usr/bin/env node
2// Publish (or update) a read-only mirror of one branch into the public Artifacts namespace that the ply Worker serves.
3//
4// env -u CLOUDFLARE_API_TOKEN node scripts/publish.mjs <repo> <source> [--branch main] [--tags]
5//
6// <source> is anything `git clone` accepts (a local path or an https URL). The branch is pushed to the same branch name,
7// with --force, so the mirror always equals the source. --tags also pushes every tag on that branch.
8//
9// Credentials: this script uses your cf login (OAuth) to create the repo if it is missing and to mint a WRITE token for
10// that one repo with a 10-minute TTL. The token reaches git only through GIT_CONFIG_* environment variables (never argv,
11// never a file, never printed), and the script revokes it when the push is done. The Worker never holds a write token: it
12// mints its own read tokens through its namespace-scoped binding.
13import { spawnSync } from 'node:child_process';
14import { mkdtempSync, rmSync } from 'node:fs';
15import { tmpdir } from 'node:os';
16import { join } from 'node:path';
17import { fileURLToPath } from 'node:url';
18import { PUBLIC_NAMESPACE } from '../src/public-namespace.ts';
20const ROOT = fileURLToPath(new URL('..', import.meta.url));
21const CF = join(ROOT, 'node_modules', '.bin', 'cf');
22const WRITE_TTL_S = 600;
23const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/;
25// Throws, so the `finally` that revokes minted tokens always runs.
26class Die extends Error {}
27// git's stderr without progress lines (they are separated by \r), last part only.
28const gitTail = text => String(text).split(/[\r\n]+/).filter(l => l.trim() && !/(Processing|Resolving|Counting|Compressing|Writing|Enumerating) objects|^remote: *$/.test(l)).join('\n').slice(-800);
29function die(msg) { throw new Die(`publish: ${msg}`); }
30process.on('uncaughtException', e => { console.error(e instanceof Die ? e.message : e); process.exit(1); });
32function parseArgs(argv) {
33 const out = { branch: 'main', tags: false, positional: [] };
34 for (let i = 0; i < argv.length; i++) {
35 const a = argv[i];
36 if (a === '--branch') out.branch = argv[++i];
37 else if (a === '--tags') out.tags = true;
38 else if (a.startsWith('--')) die(`unknown flag ${a}`);
39 else out.positional.push(a);
40 }
41 return out;
44function cf(args, { allowFail = false } = {}) {
45 const r = spawnSync(CF, args, { encoding: 'utf8', env: { ...process.env, CF_SEND_TELEMETRY: 'false' } });
46 if (r.status !== 0 && !allowFail) die(`cf ${args.slice(0, 4).join(' ')} … exited ${r.status}: ${(r.stderr || r.stdout).slice(0, 400)}`);
47 return r;
50function json(text) {
51 const start = text.indexOf('{');
52 if (start < 0) die('cf did not print JSON');
53 return JSON.parse(text.slice(start));
56function git(args, { env = {}, cwd } = {}) {
57 const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 1 << 28, env: { ...process.env, GIT_TERMINAL_PROMPT: '0', ...env } });
58 if (r.status !== 0) die(`git ${args[0]} exited ${r.status}: ${gitTail(r.stderr || r.error || '')}`);
59 return r.stdout;
62const { positional: [repo, source], branch, tags } = parseArgs(process.argv.slice(2));
63if (!repo || !source) die('usage: node scripts/publish.mjs <repo> <source> [--branch main] [--tags]');
64if (!NAME.test(repo) || repo.includes('..') || /\.git$/i.test(repo)) die('repo name: a letter or digit, then letters, digits, . _ - (no .., no .git suffix)');
65if (!/^[A-Za-z0-9][A-Za-z0-9._/-]*$/.test(branch) || branch.includes('..')) die('bad branch name');
67// 1. The repo exists, or is created with this branch as its default.
68const got = cf(['artifacts', 'namespaces', 'repos', 'get', repo, '--namespace', PUBLIC_NAMESPACE], { allowFail: true });
69let remote;
70if (got.status === 0) remote = json(got.stdout).remote;
71else {
72 const created = json(cf(['artifacts', 'namespaces', 'repos', 'create', PUBLIC_NAMESPACE, '--name', repo, '--default-branch', branch, '--description', `Read-only mirror (${source.replace(/\/\/[^@/]*@/, '//')})`]).stdout);
73 remote = created.remote;
74 console.log(`created ${PUBLIC_NAMESPACE}/${repo}`);
75 // The create response carries an initial write token (24 h) but not its id. It is not used: revoke every active
76 // write token on the new repo so none outlives this run.
77 const listed = cf(['artifacts', 'namespaces', 'repos', 'tokens', 'list', '--namespace', PUBLIC_NAMESPACE, '--name', repo, '--state', 'active'], { allowFail: true });
78 if (listed.status === 0) {
79 for (const t of JSON.parse(listed.stdout.slice(listed.stdout.indexOf('[')))) {
80 if (t.scope === 'write') cf(['artifacts', 'namespaces', 'tokens', 'revoke', String(t.id), '--namespace', PUBLIC_NAMESPACE, '--force'], { allowFail: true });
81 }
82 }
84if (!/^https:\/\/[a-z0-9]+\.artifacts\.cloudflare\.net\/git\//.test(String(remote))) die('unexpected remote from cf');
86// 2. A local bare copy of just that branch (and its tags when asked).
87const work = mkdtempSync(join(tmpdir(), 'ply-publish-'));
88const tokenIds = [];
89let current = null; // { token, at }
90// A write token for this repo only, 10 minutes. Re-minted after 8 minutes for a long chunked publish.
91function writeToken() {
92 if (current && Date.now() - current.at < (WRITE_TTL_S - 120) * 1000) return current.token;
93 const minted = json(cf(['artifacts', 'namespaces', 'tokens', 'create', PUBLIC_NAMESPACE, '--repo', repo, '--scope', 'write', '--ttl', String(WRITE_TTL_S)]).stdout);
94 tokenIds.push(minted.id);
95 if (!minted.plaintext) die('cf returned no token');
96 current = { token: String(minted.plaintext), at: Date.now() };
97 return current.token;
99// Push. The Authorization header goes to git through the environment only. Returns null or git's stderr.
100function push(refspecs, cwd) {
101 const env = { GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'http.extraHeader', GIT_CONFIG_VALUE_0: `Authorization: Bearer ${writeToken()}` };
102 const r = spawnSync('git', ['push', '--quiet', remote, ...refspecs], { cwd, encoding: 'utf8', maxBuffer: 1 << 28, env: { ...process.env, GIT_TERMINAL_PROMPT: '0', ...env } });
103 return r.status === 0 ? null : String(r.stderr || 'push failed');
105// Artifacts refuses a push over 64 MiB or 50,000 objects (developers.cloudflare.com/artifacts/platform/limits).
106const TOO_BIG = /maximum size|too.large|limit is|exceeds|too many objects|413|object limit|object count/i;
107try {
108 const cwd = join(work, 'src.git');
109 git(['clone', '--quiet', '--bare', '--single-branch', '--branch', branch, ...(tags ? [] : ['--no-tags']), source, cwd]);
110 const sha = git(['rev-parse', `refs/heads/${branch}`], { cwd }).trim();
111 const target = `refs/heads/${branch}`;
112 let err = push([`+${target}:${target}`], cwd);
113 if (err) console.log(`full push refused: ${gitTail(err)}`);
114 if (err && TOO_BIG.test(err)) {
115 // Over the per-push limit: walk the first-parent chain oldest → newest and push prefixes of history, each one a
116 // fast-forward of the last, halving the stride when a push is still too big.
117 const chain = git(['rev-list', '--first-parent', '--reverse', target], { cwd }).trim().split('\n');
118 // Resume from what the remote already has, so a re-run never rewinds the branch.
119 const auth = { GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'http.extraHeader', GIT_CONFIG_VALUE_0: `Authorization: Bearer ${writeToken()}` };
120 const have = (git(['ls-remote', remote, target], { cwd, env: auth }).split('\t')[0] ?? '').trim();
121 let done = have ? chain.indexOf(have) : -1, stride = Math.max(1, Math.floor(chain.length / 8)), pushes = 0;
122 while (done < chain.length - 1) {
123 const next = Math.min(chain.length - 1, done + stride);
124 const e = push([`+${chain[next]}:${target}`], cwd);
125 if (e) {
126 if (!TOO_BIG.test(e) || stride === 1) die(`chunked push failed at ${next}/${chain.length}: ${gitTail(e)}`);
127 stride = Math.max(1, Math.floor(stride / 2));
128 continue;
129 }
130 done = next; pushes++;
131 console.log(`pushed ${done + 1}/${chain.length} first-parent commits (${pushes} pushes)`);
132 stride = Math.ceil(stride * 1.5);
133 }
134 err = null;
135 }
136 if (err) die(`push failed: ${gitTail(err)}`);
137 if (tags) { const e = push(['+refs/tags/*:refs/tags/*'], cwd); if (e) die(`tag push failed: ${e.slice(0, 300)}`); }
138 console.log(`published ${PUBLIC_NAMESPACE}/${repo} ${branch} = ${sha}`);
139 console.log(`view: https://ply.ivar.workers.dev/r/${repo}`);
140} finally {
141 rmSync(work, { recursive: true, force: true });
142 // Revocation is eventually consistent (cf: a revoke within ~1 s of create can 404), so retry briefly.
143 for (const tokenId of tokenIds) {
144 let ok = false;
145 for (let i = 0; i < 5 && !ok; i++) {
146 ok = cf(['artifacts', 'namespaces', 'tokens', 'revoke', String(tokenId), '--namespace', PUBLIC_NAMESPACE, '--force'], { allowFail: true }).status === 0;
147 if (!ok) spawnSync('sleep', ['1']);
148 }
149 console.log(ok ? `write token ${tokenId} revoked` : `write token ${tokenId} not revoked; it expires in ${WRITE_TTL_S / 60} min`);
150 }
Diagnostics4 git calls · 29 ms
CallStatusSizeTime029 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit8 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=5
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=5
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020042 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=42585
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=262b639aa1b258fc1777c3a66ba8f811df3a3da2
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=42585
git.pkt.payload.max=42490
git.sideband.band1.bytes=42489
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=42489
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=7dbf576d97429022615f79aa46b440d9da603496
git.pack.full=61
Open pack map
fetch2003.7 KiB · hit19 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=19
git.response.bytes=3780
git.response.cache=HIT
git.response.duration_ms=19
git.fetch.wants=da9df8791f6a03bdfc70f54b94f92064482761ec
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3780
git.pkt.payload.max=3759
git.sideband.band1.bytes=3758
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
262b639aa1b258fc1777c3a66ba8f811df3a3da2
cf-ray
a5e1c31d08cf36cd
Request
836172c5…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format