import { spawnSync } from 'node:child_process' import { startClientHealth, stopClientHealth } from './client-health.ts' import { readFile } from 'node:fs/promises' import { hostname } from 'node:os' import { basename, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { ROOT } from './cf.ts' import { loadConfig, publicConfig, type CdConfig } from './config.ts' import { splitPolicy } from './policy.ts' import { writeDebrief } from './debrief.ts' import { VERSION_ID } from './deploy-plan.ts' import { appendFalsePositive, RunLog, withRun } from './log.ts' import { runPipeline } from './pipeline.ts' import { binaryNodeVersion, checkNode, NODE_IMAGE_BIN, nodeSatisfies, pathWithNodeBin, selectNode } from './preflight.ts' import { errorMessage, run } from './proc.ts' import { unfinishedRunDir } from './reconcile.ts' import { pruneRuns, retentionLine } from './retention.ts' import { checkoutReason, isReleaseId, readCheckout, releaseId, releasePair, type ReleaseId } from './release.ts' import { reportLedger } from './report.ts' import { createRunDir, RUN_ID, writeJson } from './run-dir.ts' import { acquireLock, realLockIo } from './lock.ts' import { readPlyPin, resolvePlySha, stageBump } from './stage-bump.ts' import { stageDeploy } from './stage-deploy.ts' import { stageRamp } from './stage-ramp.ts' import { stageRollback } from './stage-rollback.ts' import { stageSmoke, stageSweep } from './stage-smoke.ts' import { stageTelemetry } from './stage-telemetry.ts' import { withCommandSink } from './trace.ts' import { readTokenFile } from './token.ts' import { parseSince } from './config.ts' export type Parsed = | { readonly kind: 'cd' readonly sha: string | null readonly noBump: boolean /** `--ramp` was passed. It is a no-op since 2026-11-08 (the ramp is the only path); the run says so. */ readonly rampFlag: boolean readonly replaceCanary: string | null } | { readonly kind: 'bump'; readonly sha: string | null } | { readonly kind: 'deploy-canary'; readonly replaceCanary: string | null } | { readonly kind: 'smoke'; readonly version: string | null; readonly expectVersion: string | null } | { readonly kind: 'telemetry'; readonly version: string; readonly since: string; readonly report: string | null } | { readonly kind: 'ramp'; readonly version: string; readonly previous: string | null; readonly verdict: string | null } | { readonly kind: 'rollback'; readonly to: string } | { readonly kind: 'report' } | { readonly kind: 'annotate'; readonly runId: string; readonly reason: string } | { readonly kind: 'debrief'; readonly runDir: string } | { readonly kind: 'sweep' } | { readonly kind: 'usage' } | { readonly kind: 'error'; readonly reason: string } function flag(argv: readonly string[], name: string): string | null { const index = argv.indexOf(`--${name}`) if (index < 0) return null const value = argv[index + 1] if (!value || value.startsWith('--')) return null return value } function uuid(value: string | null): string | null { if (value === null) return null if (!VERSION_ID.test(value)) return null return value } /** `--no-bump` may name a sha only when that sha is already this checkout's ply pin. */ export function noBumpSha(sha: string, pin: string | null): string | null { if (sha === pin) return null return '--no-bump deploys the checkout; --sha must equal the current pin' } function parseCd(argv: readonly string[], pin: string | null): Parsed { let sha: string | null = null let noBump = false let rampFlag = false let replaceCanary: string | null = null for (let i = 0; i < argv.length; i++) { const arg = argv[i] ?? '' if (arg === '--no-bump') { noBump = true continue } if (arg === '--ramp') { rampFlag = true continue } if (arg === '--replace-canary') { const value = uuid(argv[i + 1] || null) if (!value) return { kind: 'error', reason: '--replace-canary needs the version id now at 0%' } replaceCanary = value i += 1 continue } if (arg === '--sha') { const value = argv[i + 1] if (!value || value.startsWith('--') || !/^[0-9a-f]{40}$/.test(value)) { return { kind: 'error', reason: '--sha must be 40 hex characters' } } sha = value i += 1 continue } return { kind: 'error', reason: arg.startsWith('--') ? `unknown option ${arg}` : `unknown command ${arg}` } } if (noBump && sha) { const mismatch = noBumpSha(sha, pin) if (mismatch) return { kind: 'error', reason: mismatch } } return { kind: 'cd', sha, noBump, rampFlag, replaceCanary } } export function parseArgs(argv: readonly string[], pin: string | null = null): Parsed { const [command, ...rest] = argv if (command === 'help' || command === '--help' || command === '-h') return { kind: 'usage' } if (!command || command === 'cd' || command.startsWith('--')) { return parseCd(command === 'cd' ? rest : argv, pin) } if (command === 'bump') { const sha = flag(rest, 'sha') if (sha !== null && !/^[0-9a-f]{40}$/.test(sha)) return { kind: 'error', reason: '--sha must be 40 hex characters' } return { kind: 'bump', sha } } if (command === 'deploy-canary') { if (rest.length === 0) return { kind: 'deploy-canary', replaceCanary: null } const replaceCanary = uuid(flag(rest, 'replace-canary')) if (!replaceCanary || rest.length !== 2) return { kind: 'error', reason: 'deploy-canary takes only --replace-canary ' } return { kind: 'deploy-canary', replaceCanary } } if (command === 'smoke') { const version = flag(rest, 'version') const expectVersion = flag(rest, 'expect-version') if (Boolean(version) === Boolean(expectVersion)) return { kind: 'error', reason: 'smoke needs --version or --expect-version' } const id = uuid(version ?? expectVersion) if (!id) return { kind: 'error', reason: 'version id must be a uuid' } return { kind: 'smoke', version: version ? id : null, expectVersion: expectVersion ? id : null } } if (command === 'telemetry') { const version = uuid(flag(rest, 'version')) const since = flag(rest, 'since') if (!version || !since) return { kind: 'error', reason: 'telemetry needs --version and --since' } return { kind: 'telemetry', version, since, report: flag(rest, 'report') } } if (command === 'ramp') { const version = uuid(flag(rest, 'version')) if (!version) return { kind: 'error', reason: 'ramp needs --version' } const previous = flag(rest, 'previous') if (previous !== null && !VERSION_ID.test(previous)) return { kind: 'error', reason: '--previous must be a uuid' } return { kind: 'ramp', version, previous, verdict: flag(rest, 'verdict') } } if (command === 'rollback') { const to = uuid(flag(rest, 'to')) if (!to) return { kind: 'error', reason: 'rollback needs --to ' } return { kind: 'rollback', to } } if (command === 'report') return { kind: 'report' } if (command === 'annotate') { const runId = rest[0] if (!runId || runId.startsWith('--')) return { kind: 'error', reason: 'annotate needs a run id' } if (!RUN_ID.test(runId)) return { kind: 'error', reason: 'run id must look like 2026-11-01T00-10-02-620Z' } const reason = flag(rest, 'false-positive') if (reason === null) return { kind: 'error', reason: 'annotate needs --false-positive "reason"' } if (!reason.trim() || /[\n\r]/.test(reason)) return { kind: 'error', reason: 'false-positive reason must be one line' } return { kind: 'annotate', runId, reason: reason.trim() } } if (command === 'debrief') { const runDir = rest[0] if (!runDir || runDir.startsWith('--')) return { kind: 'error', reason: 'debrief needs a run directory' } return { kind: 'debrief', runDir } } if (command === 'sweep') { if (rest.length > 0) return { kind: 'error', reason: 'sweep takes no arguments' } return { kind: 'sweep' } } return { kind: 'error', reason: `unknown command ${command}` } } export function usageText(): string { return [ 'usage: npm run cd -- [--policy ] [command]', ' cd [--sha <40 hex>] [--no-bump] [--replace-canary ]', ' [--sha <40 hex>] [--no-bump] [--replace-canary ]', ' 0% canary, pinned smoke + telemetry, then the stepped ramp in the policy (steps) to 100%;', ' live smoke + telemetry + visual smoke during the 100% step (auto-rollback). The ramp is the only path:', ' --ramp is accepted and ignored. --replace-canary lets this run replace that version at 0%.', ' One run at a time: the deployment lock (PLY_CD_LOCK, ~/.local/state/ply-cd/deploy.lock) refuses a second.', ' bump [--sha <40 hex>]', ' deploy-canary [--replace-canary ]', ' smoke --version | --expect-version ', ' telemetry --version --since --report smoke.json', ' ramp --version [--previous ] [--verdict telemetry.json]', ' rollback --to ', ' report', ' annotate --false-positive "reason"', ' debrief ', ' sweep', ].join('\n') } function finish(log: (line: string) => void, code: number, line: string): number { log(line) return code } async function toolLine(cmd: string, args: readonly string[]): Promise { const result = await run(cmd, args, { cwd: ROOT }) const line = (result.stdout || result.stderr).trim().split('\n')[0] ?? '' return result.code === 0 && line ? line : `${cmd} unavailable` } async function identity(config: CdConfig) { const pkg = JSON.parse(await readFile(join(ROOT, 'package.json'), 'utf8')) as { dependencies?: { ply?: string } } const spec = pkg.dependencies?.ply ?? '' const pin = /#([0-9a-f]{40})$/.exec(spec) const plyWeb = await toolLine('git', ['rev-parse', 'HEAD']) return { config: publicConfig(config), tools: { node: process.version, cf: await toolLine(join(ROOT, 'node_modules', '.bin', 'cf'), ['--version']), git: await toolLine('git', ['--version']), }, plySha: pin?.[1] ?? null, plyWebSha: plyWeb === 'git unavailable' ? 'unknown' : plyWeb, host: hostname(), } } async function commandRelease(parsed: Parsed): Promise { if (parsed.kind !== 'cd' && parsed.kind !== 'deploy-canary') return null const checkout = await readCheckout(ROOT) if (!checkout.ok) return `cd: harness_fault ${checkoutReason(checkout)}` let ply: string | null try { if (parsed.kind === 'deploy-canary') ply = await readPlyPin() else if (parsed.noBump) ply = parsed.sha ?? await readPlyPin() else ply = await resolvePlySha(parsed.sha) } catch (error) { return `cd: harness_fault bad-ply-sha: ${errorMessage(error)}` } if (!ply) return 'cd: harness_fault bad-ply-sha: package.json has no ply pin' const made = releaseId(ply, checkout.web) if (!isReleaseId(made)) return `cd: harness_fault ${made.kind}: ply and ply-web shas must be 40 hex` return made } /** Leave the process on a Node that satisfies `package.json` engines when this one does not. */ function reexecNode(): number | null { if (process.env.PLY_CD_NODE_REEXEC === '1' || nodeSatisfies(process.version)) return null const candidates = [] const preferred = process.env.PLY_NODE if (preferred) candidates.push({ bin: preferred, version: binaryNodeVersion(preferred) }) candidates.push({ bin: NODE_IMAGE_BIN, version: binaryNodeVersion(NODE_IMAGE_BIN) }) const better = selectNode(process.version, candidates) if (!better) return null const child = spawnSync(better, [...process.execArgv, ...process.argv.slice(1)], { stdio: 'inherit', env: { ...process.env, PLY_CD_NODE_REEXEC: '1', PATH: pathWithNodeBin(better, process.env.PATH) }, }) return child.status === null ? 1 : child.status } export async function cli(argv: readonly string[], env: NodeJS.ProcessEnv, log: (line: string) => void = console.log): Promise { const switched = reexecNode() if (switched !== null) return switched const nodeError = checkNode(process.version) if (nodeError) return finish(log, 1, `cd: harness_fault preflight: ${nodeError}`) const split = splitPolicy(argv) if ('error' in split) return finish(log, 2, split.error) const parsed = parseArgs(split.argv, await readPlyPin()) if (parsed.kind === 'usage') return finish(log, 2, usageText()) if (parsed.kind === 'error') return finish(log, 2, parsed.reason) let config: CdConfig try { config = loadConfig(env, split.policyPath ?? undefined) } catch (error) { return finish(log, 2, errorMessage(error)) } if (parsed.kind === 'report') { log(await reportLedger(config.ledgerPath)) return 0 } if (parsed.kind === 'annotate') { try { await appendFalsePositive(config.ledgerPath, parsed.runId, parsed.reason) return finish(log, 0, `annotated ${parsed.runId} as a false positive`) } catch (error) { return finish(log, 1, `cd: FAIL annotate: ${errorMessage(error)}`) } } if (parsed.kind === 'debrief') { try { const path = await writeDebrief(resolve(parsed.runDir)) return finish(log, 0, `debrief: ${path}`) } catch (error) { return finish(log, 1, `cd: FAIL debrief: ${errorMessage(error)}`) } } const opened = await commandRelease(parsed) if (typeof opened === 'string') return finish(log, 1, opened) const release = opened const needsToken = parsed.kind === 'cd' || parsed.kind === 'smoke' || parsed.kind === 'ramp' let token = '' if (needsToken) { try { token = await readTokenFile(config.tokenFile) } catch (error) { return finish(log, 1, `cd: harness_fault preflight: ${errorMessage(error)}`) } } // Every command that writes the deployment holds the lock for its whole run (scripts/cd/lock.ts). const writes = parsed.kind === 'cd' || parsed.kind === 'deploy-canary' || parsed.kind === 'ramp' || parsed.kind === 'rollback' const lock = writes ? await acquireLock(config.lockPath, { command: ['cd', ...argv].join(' '), cwd: ROOT, pid: process.pid }, realLockIo) : null if (lock && !lock.ok) return finish(log, 1, `cd: state_conflict lock: ${lock.reason}`) if (lock && lock.ok) { log(`lock: ${lock.path} (pid ${lock.holder.pid})`) if (lock.tookOver) log(`lock: took over a stale lock from pid ${lock.tookOver.pid} (${lock.tookOver.startedAt}), which is no longer running`) } try { return await runLocked(parsed, config, token, release, log) } finally { if (lock && lock.ok) await lock.release() } } async function runLocked( parsed: Exclude, config: CdConfig, token: string, release: ReleaseId | null, log: (line: string) => void, ): Promise { if (parsed.kind === 'cd' && parsed.rampFlag) log('cd: --ramp is ignored: the stepped ramp is the only promotion path since 2026-11-08') const existing = parsed.kind === 'cd' && release ? await unfinishedRunDir(config.runRoot, release) : null try { log(retentionLine(await pruneRuns(config.runRoot, existing ? [basename(existing)] : []))) } catch (error) { // Retention never blocks a deploy. The next run tries again. log(`retention: skipped: ${errorMessage(error)}`) } const runDir = existing ?? await createRunDir(config.runRoot) const runLog = existing ? await RunLog.reopen(runDir, config.ledgerPath) : await RunLog.open(runDir, config.ledgerPath, basename(runDir)) log(`run: ${runDir}`) if (existing) log(`resume: ${runDir}`) return withRun(runLog, () => withCommandSink(note => runLog.noteCommand(note), async () => { let code = 1 let reason: string | null = 'run ended before finish' try { if (token) runLog.conceal(token) if (existing && release) { await runLog.resumed(release.ply, releasePair(release)) } else { const started = await identity(config) const pair = release ? releasePair(release) : null const recorded = { ...started, ...(pair ? { plySha: pair.ply, plyWebSha: pair.web, release: pair } : { release: null }), config: pair ? { ...started.config, sha: pair.ply, release: pair } : started.config, } await runLog.started(recorded) } await runLog.recordPolicy({ sha256: config.policySha256, path: config.policyPath, body: config.policyBody, release: release ? releasePair(release) : null, }) // Watch this process's own clocks for the whole run (client-health.ts): a stalled box is never a product verdict. startClientHealth() const outcome = await dispatch(parsed, config, token, runDir, runLog, log, release) code = outcome.code reason = outcome.reason return code } catch (error) { reason = errorMessage(error) runLog.patch({ fault: 'harness_fault' }) await runLog.error(reason, { stage: 'run', fault: 'harness_fault' }) log(`cd: harness_fault run: ${reason}`) return 1 } finally { const stalls = stopClientHealth() if (stalls.length > 0) log(`client: ${stalls.length} stall(s) this run: ${stalls.map(stall => `${new Date(stall.from).toISOString()} ${stall.ms} ms`).join(', ')}`) if (!runLog.finished) await runLog.finish(code, code === 0 ? null : reason) if (!runLog.sealed) await runLog.seal() } })) } type Outcome = { readonly code: number; readonly reason: string | null } async function dispatch( parsed: Exclude, config: CdConfig, token: string, runDir: string, runLog: RunLog, log: (line: string) => void, release: ReleaseId | null, ): Promise { if (parsed.kind === 'cd') { if (!release) return { code: 1, reason: 'cd: harness_fault bad-ply-sha: release was not read' } const exit = await runPipeline(config, { token, sha: parsed.sha, noBump: parsed.noBump, replaceCanary: parsed.replaceCanary, release, log, runDir, runLog }) return { code: exit.code, reason: exit.reason } } if (parsed.kind === 'bump') { const result = await stageBump(config, parsed.sha) await writeJson(runDir, 'bump.json', result) if (result.kind === 'fail') { runLog.patch({ fault: 'harness_fault' }) const line = `cd: harness_fault bump: ${result.reason}` log(line) return { code: 1, reason: line } } const line = result.value.committed ? `cd: PASS bump ${result.value.sha}` : `cd: PASS bump unchanged ${result.value.sha}` log(line) return { code: 0, reason: null } } if (parsed.kind === 'deploy-canary') { if (!release) return { code: 1, reason: 'cd: harness_fault bad-ply-sha: release was not read' } const result = await stageDeploy(config, release, { replaceCanary: parsed.replaceCanary }) await writeJson(runDir, 'deploy.json', result) if (result.kind === 'fail') { runLog.patch({ fault: result.fault }) const line = `cd: ${result.fault === 'worker' ? 'FAIL' : result.fault} deploy-canary: ${result.reason}` log(line) return { code: 1, reason: line } } log(`version: ${result.value.version}`) log(`previous: ${result.value.previous}`) log(`cd: PASS deploy-canary ${result.value.version}`) return { code: 0, reason: null } } if (parsed.kind === 'smoke') { const version = parsed.version ?? parsed.expectVersion const result = await stageSmoke(config, { version, pin: parsed.version !== null, token, runDir }) if (result.kind === 'fail') { runLog.patch({ fault: result.fault }) const line = `cd: ${result.fault === 'worker' ? 'FAIL' : result.fault} smoke: ${result.reason}` log(line) return { code: 1, reason: line } } log(`cd: PASS smoke ${version}`) return { code: 0, reason: null } } if (parsed.kind === 'telemetry') { let since = 0 try { since = parseSince(parsed.since) } catch (error) { const line = errorMessage(error) log(line) return { code: 2, reason: line } } if (!parsed.report) { log('telemetry needs --report') return { code: 2, reason: 'telemetry needs --report' } } const result = await stageTelemetry(config, { version: parsed.version, since, reportPath: parsed.report, runDir, live: false }) if (result.kind === 'fail') { const line = `cd: ${result.fault === 'worker' ? 'FAIL' : result.fault} telemetry: ${result.reason}` log(line) return { code: 1, reason: line } } log(result.value.line) log(`cd: PASS telemetry ${parsed.version}`) return { code: 0, reason: null } } if (parsed.kind === 'sweep') { const result = await stageSweep(config, runDir) if (result.kind === 'unlistable') { runLog.patch({ fault: 'harness_fault' }) const line = `cd: harness_fault sweep: ${result.detail}` log(line) return { code: 1, reason: line } } if (result.leaks.length > 0) log(`cleanup_leak: ${result.leaks.join(' ')}`) const line = `cd: PASS sweep deleted ${result.deleted.length}` log(line) return { code: 0, reason: null } } if (parsed.kind === 'ramp') { const result = await stageRamp(config, { version: parsed.version, previous: parsed.previous, verdictPath: parsed.verdict, runDir, token, log, }) if (result.kind === 'fail') { const line = `cd: ${result.fault === 'worker' ? 'FAIL' : result.fault} ramp: ${result.reason}` log(line) return { code: 1, reason: line } } log(`version: ${result.value.version}`) log(`previous: ${result.value.previous}`) log(`cd: PASS ramp ${result.value.version}`) return { code: 0, reason: null } } const result = await stageRollback(config, parsed.to) await writeJson(runDir, 'rollback.json', result) if (result.kind === 'fail') { const line = `cd: FAIL rollback: ${result.reason}` log(line) return { code: 1, reason: line } } log(`cd: PASS rollback ${parsed.to}`) return { code: 0, reason: null } } const entry = process.argv[1] if (entry && resolve(entry) === fileURLToPath(import.meta.url)) { cli(process.argv.slice(2), process.env).then(code => process.exit(code)) }