main
ply4 git calls427 ms47 KiB2 cachedprotocol v2
430 lines19 KiBRaw
1import { assertNever } from '../../src/never.ts'
2import { readFile, appendFile } from 'node:fs/promises'
3import { CLIENT_MAX_EXTENSION_MS } from './baseline.ts'
4import { clientHealth } from './client-health.ts'
5import { join } from 'node:path'
6import { none, orNull, some, type Option } from '../../src/option.ts'
7import type { CdConfig } from './config.ts'
8import { isTrafficSplit, sameSplit, trafficSplit, type VersionShare } from './deploy-plan.ts'
9import { notSpanFilter, queryCapturedSliced, telemetryQueryRetry } from './o11y.ts'
10import { errorMessage } from './proc.ts'
11import { activeLog, BLOB_DIR } from './log.ts'
12import { readPassVerdict, runRamp, type HarnessTraffic, type LatencySample, type Observation, type RampLoopResult, type RampState } from './ramp.ts'
13import { fail, pass, type StageResult } from './result.ts'
14import type { RolloutPolicy } from './policy.ts'
15import { accountProbeLoss, classifyProbe, probeLossRecord, rayOf, type ProbeOutcome } from './probe.ts'
16import { runProbeSchedule } from './probe-schedule.ts'
17import { writeJson } from './run-dir.ts'
18import { routedShare } from './slo.ts'
19import { percentile, queryTo, rayIdOf, rayKey, scriptVersionId, serviceFilter, statsFromEvents, type VersionStats } from './telemetry.ts'
20import { asRecord } from './json.ts'
21import { applySplit, readShares, rollbackTo } from './traffic.ts'
22import { armStep, beginRamp, endRamp, HarnessFault, noteStepApplied } from './traffic-gate.ts'
24export type RampValue = {
25 readonly version: string
26 readonly previous: string
27 readonly lines: readonly string[]
30/** A ramp state whose unjudged step `settleHarnessFault` settled (ramp.ts HarnessTraffic). */
31export type UnjudgedState = Extract<RampState, { readonly traffic: HarnessTraffic }>
33/**
34 * The stage result of an unjudged step (ply-web#145). `rolledBack` is true only when the rollback landed; the fault is
35 * the step's own outcome, never `worker`, because the version was not judged. The state's reason already says where
36 * traffic is, including a failed rollback or hold write, so it is not repeated here.
37 */
38export function unjudgedFail(state: UnjudgedState): StageResult<RampValue> {
39 const rolledBack = state.traffic === 'rolled_back'
40 switch (state.kind) {
41 case 'held':
42 return fail('ramp', state.reason, rolledBack, state.outcome)
43 case 'stopped':
44 return fail('ramp', state.reason, rolledBack, state.fault)
45 default:
46 return assertNever(state)
47 }
50/**
51 * The stage result of a finished ramp loop (ply-web#145): complete passes; a policy hold fails as `worker`, not rolled
52 * back; an unjudged hold or harness_fault stop goes through `unjudgedFail`; a state_conflict stop is not rolled back;
53 * a rollback is rolledBack unless its write failed, which names `previous` and the error and is not rolledBack.
54 */
55export function rampStageResult(
56 result: Pick<RampLoopResult, 'state' | 'lines' | 'rollbackError'>,
57 version: string,
58 previous: string,
59): StageResult<RampValue> {
60 const state = result.state
61 if (state.kind === 'complete') return pass('ramp', { version, previous, lines: result.lines })
62 if (state.kind === 'held') {
63 if (!('traffic' in state)) return fail('ramp', state.reason, false)
64 return unjudgedFail(state)
65 }
66 if (state.kind === 'stopped') {
67 if (state.fault === 'harness_fault') return unjudgedFail(state)
68 return fail('ramp', state.reason, false, state.fault)
69 }
70 const reason = state.kind === 'rolled-back' ? state.reason : 'ramp stopped'
71 if (result.rollbackError) return fail('ramp', `${reason}; rollback to ${previous} failed: ${result.rollbackError}`)
72 return fail('ramp', reason, true)
75function eventsFor(events: readonly unknown[], version: string): unknown[] {
76 return events.filter(event => scriptVersionId(event) === version)
79/** An event whose cf-ray is one of `rays` (a probe's response header against the invocation's `$metadata.rayId`). */
80function rayIn(event: unknown, rays: ReadonlySet<string>): boolean {
81 const rec = asRecord(event)
82 const ray = rec ? rayIdOf(rec) : null
83 return ray !== null && rays.has(rayKey(ray))
86/**
87 * The window's stats with wall and CPU read from `sample` only (ramp.ts LatencySample): counts, errors, 5xx, internal
88 * calls and routed share stay those of every event. `excluded` is how many external invocations the latency left out;
89 * `landed` is how many are in the sample (the probes that reached the telemetry), which the ingest wait counts.
90 */
91export function scopedStats(events: readonly unknown[], sample: LatencySample): { readonly stats: VersionStats; readonly excluded: number; readonly landed: number } {
92 const whole = statsFromEvents(events, [])
93 if (sample.kind === 'all') return { stats: whole, excluded: 0, landed: whole.invocations }
94 const rays = new Set(sample.rays.map(rayKey))
95 const timed = statsFromEvents(events.filter(event => rayIn(event, rays)), [])
96 return { stats: { ...whole, wallMs: timed.wallMs, cpuMs: timed.cpuMs }, excluded: whole.wallMs.length - timed.wallMs.length, landed: timed.invocations }
99export async function observe(config: CdConfig, version: string, previous: string, from: number, to: number, percent: number, sample: LatencySample): Promise<Observation> {
100 const service = serviceFilter(config.worker)
101 const query = { view: 'events' as const, filters: [service] }
102 try {
103 const retry = telemetryQueryRetry(config.policy)
104 // Invocations and logs only, in parallel 10 s slices. The unscoped span query paged past TELEMETRY_MAX_PAGES under
105 // load (run 2026-11-06T21-37-46-183Z); spans only fed fetchMs, which gates nothing here.
106 const all = await queryCapturedSliced('events', from, to, [service, notSpanFilter()], retry)
107 const spans = { events: [] as unknown[], query: all.query }
108 const scopedNext = scopedStats(eventsFor(all.events, version), sample)
109 const scopedPrior = scopedStats(eventsFor(all.events, previous), sample)
110 const scopedAll = scopedStats(all.events, sample)
111 const next = scopedNext.stats
112 const prior = scopedPrior.stats
113 const aggregate = scopedAll.stats
114 const share = routedShare(next.invocations, aggregate.invocations, percent, config.policy.split.z)
115 const log = activeLog()
116 if (!log) throw new HarnessFault('ramp observation with no active log')
117 const ref = await log.observation({
118 measurement: {
119 source: 'ramp-step',
120 available: true,
121 percent,
122 targetPercent: percent,
123 nextRequests: next.invocations,
124 previousRequests: prior.invocations,
125 aggregateRequests: aggregate.invocations,
126 routedNewShare: share.share === null ? null : 100 * share.share,
127 toleranceLow: share.low === null ? null : 100 * share.low,
128 toleranceHigh: share.high === null ? null : 100 * share.high,
129 latency: config.policy.gates.latency.mode,
130 nextWallP99: percentile(next.wallMs, 0.99),
131 previousWallP99: percentile(prior.wallMs, 0.99),
132 nextCpuP99: percentile(next.cpuMs, 0.99),
133 previousCpuP99: percentile(prior.cpuMs, 0.99),
134 latencySample: sample.kind,
135 latencyProbeRays: sample.kind === 'probes' ? sample.rays.length : 0,
136 latencySamples: { next: next.wallMs.length, previous: prior.wallMs.length, aggregate: aggregate.wallMs.length },
137 latencyExcluded: { next: scopedNext.excluded, previous: scopedPrior.excluded, aggregate: scopedAll.excluded },
138 internalInvocations: {
139 next: next.internal,
140 previous: prior.internal,
141 aggregate: aggregate.internal,
142 },
143 },
144 query: { events: all.query, spans: spans.query },
145 from,
146 to,
147 // The pages are the command captures already; cite them by digest instead of embedding a second, escaped copy.
148 raw: JSON.stringify({ pages: all.pages.map(digest => `${BLOB_DIR}/${digest}.gz`) }),
149 })
150 return { kind: 'sample', next, previous: prior, aggregate, landed: scopedAll.landed, ref }
151 } catch (error) {
152 if (error instanceof HarnessFault) throw error
153 const reason = `telemetry unavailable: ${errorMessage(error)}`
154 const log = activeLog()
155 if (!log) throw new HarnessFault('ramp observation with no active log')
156 const ref = await log.observation({
157 measurement: { source: 'ramp-step', available: false, percent, targetPercent: percent, reason },
158 query,
159 from,
160 to,
161 raw: JSON.stringify({ error: reason }),
162 })
163 return { kind: 'unavailable', reason, ref }
164 }
167function versionOf(header: string | null): Option<string> {
168 if (header === null || header.length === 0) return none()
169 return some(header)
172/** Unpinned. A policy timeout is `timed_out`. Any other throw is a transport loss. */
173async function probeCall(
174 id: string,
175 method: string,
176 url: string,
177 headers: Record<string, string>,
178 timeoutMs: number,
179 parent: AbortSignal,
180): Promise<ProbeOutcome> {
181 const log = activeLog()
182 const started = Date.now()
183 const signal = AbortSignal.any([AbortSignal.timeout(timeoutMs), parent])
184 try {
185 const response = await fetch(url, { method, headers, signal })
186 const latencyMs = Date.now() - started
187 const version = versionOf(response.headers.get('x-ply-version'))
188 const outcome = classifyProbe(id, { kind: 'response', status: response.status, ray: rayOf(response.headers.get('cf-ray')), latencyMs, version })
189 if (outcome.kind === 'responded' && log) {
190 await log.request({
191 method,
192 url,
193 status: outcome.status,
194 version: response.headers.get('x-ply-version'),
195 latencyMs,
196 cfRay: orNull(outcome.ray),
197 requestId: response.headers.get('x-ply-request-id'),
198 artifactsRay: response.headers.get('x-artifacts-ray'),
199 }, 'ramp')
200 }
201 return outcome
202 } catch (error) {
203 const name = error instanceof Error ? error.name : 'Error'
204 if (name === 'TimeoutError') return classifyProbe(id, { kind: 'timeout' })
205 const message = error instanceof Error ? error.message : 'probe failed'
206 return classifyProbe(id, { kind: 'thrown', name, message })
207 }
210async function dwell(
211 token: string,
212 baseUrl: string,
213 ms: number,
214 probe: RolloutPolicy['probe'],
215): Promise<readonly ProbeOutcome[]> {
216 const live = probe.calls.filter(call => !call.auth || token.length > 0)
217 if (probe.rps <= 0 || live.length === 0) {
218 await new Promise(resolve => setTimeout(resolve, ms))
219 return []
220 }
221 const controller = new AbortController()
222 return runProbeSchedule({
223 clock: {
224 now: () => Date.now(),
225 sleep: wait => new Promise(resolve => setTimeout(resolve, wait)),
226 },
227 dwellMs: ms,
228 periodMs: 1000 / probe.rps,
229 maxInFlight: probe.maxInFlight,
230 batch: live.length,
231 graceMs: probe.timeoutMs,
232 health: clientHealth(),
233 extension: { maxMs: CLIENT_MAX_EXTENSION_MS, minReal: probe.minSamples },
234 issue: seq => {
235 const call = live[(seq - 1) % live.length]
236 if (!call) throw new Error('probe call missing')
237 const url = `${baseUrl}${call.path}`
238 const id = `${seq} ${call.method} ${url}`
239 const headers: Record<string, string> = call.auth ? { authorization: `Bearer ${token}` } : {}
240 return { id, promise: probeCall(id, call.method, url, headers, probe.timeoutMs, controller.signal) }
241 },
242 shed: seq => ({ kind: 'shed', id: `${seq} shed` }),
243 stop: () => {
244 controller.abort(new DOMException('probe dwell deadline', 'TimeoutError'))
245 },
246 })
249export async function stageRamp(
250 config: CdConfig,
251 opts: {
252 readonly version: string
253 readonly previous: string | null
254 readonly verdictPath: string | null
255 readonly runDir: string
256 readonly token: string
257 readonly log: (line: string) => void
258 /** When set, the version is already at this ramp percent. Do not write it again. */
259 readonly fromPercent?: number
260 },
261 /** The schedule (the policy's steps), and a hook called once each step's split is in place: the pipeline starts the live checks at 100%. */
262 schedule: { readonly policy: RolloutPolicy; readonly onApplied: (percent: number) => void } = { policy: config.policy, onApplied: () => {} },
263): Promise<StageResult<RampValue>> {
264 const rollout = schedule.policy
265 // Traffic already on a ramp step means an earlier run opened the canary gate.
266 if (opts.fromPercent === undefined) {
267 if (!opts.verdictPath) return fail('ramp', 'ramp needs a PASS verdict file (--verdict); traffic was not changed')
268 let verdictText = ''
269 try {
270 verdictText = await readFile(opts.verdictPath, 'utf8')
271 } catch {
272 return fail('ramp', `cannot read ${opts.verdictPath}; traffic was not changed`)
273 }
274 const verdict = readPassVerdict(verdictText, opts.version)
275 if (!verdict.ok) return fail('ramp', `${verdict.reason}; traffic was not changed`)
276 }
277 let previous = opts.previous
278 let expected: VersionShare[]
279 const fromPercent = opts.fromPercent
280 try {
281 const shares = await readShares(config.worker)
282 if (fromPercent !== undefined) {
283 const mine = shares.find(share => share.versionId === opts.version)
284 const beside = shares.filter(share => share.versionId !== opts.version)
285 const other = beside[0]
286 if (!mine || !other || shares.length !== 2 || mine.percentage !== fromPercent || other.percentage !== 100 - fromPercent) {
287 return fail('ramp', `expected ${opts.version} at ${fromPercent}% beside the previous version; traffic was not changed`)
288 }
289 if (previous && previous !== other.versionId) {
290 return fail('ramp', `expected ${previous} beside ${opts.version}; traffic was not changed`)
291 }
292 previous = other.versionId
293 } else {
294 const live = shares.find(share => share.percentage === 100)
295 const dark = shares.find(share => share.versionId === opts.version)
296 if (!live) return fail('ramp', 'no version at 100%; traffic was not changed')
297 if (!previous) previous = live.versionId
298 if (live.versionId !== previous || live.percentage !== 100) {
299 return fail('ramp', `expected ${previous} at 100% before the ramp; traffic was not changed`)
300 }
301 if (!dark || dark.percentage !== 0 || shares.length !== 2) {
302 return fail('ramp', `expected ${opts.version} at 0% beside ${previous}; traffic was not changed`)
303 }
304 }
305 expected = shares.map(share => ({ versionId: share.versionId, percentage: share.percentage }))
306 } catch (error) {
307 return fail('ramp', `${errorMessage(error)}; traffic was not changed`)
308 }
309 if (!previous) return fail('ramp', 'no previous version; traffic was not changed')
310 const target = previous
311 const steps = fromPercent === undefined
312 ? rollout.steps
313 : rollout.steps.filter(step => step.percent >= fromPercent)
314 if (fromPercent !== undefined && steps.length === 0) {
315 return fail('ramp', `schedule has no step at or above ${fromPercent}%`)
316 }
317 const policy = { ...rollout, steps }
318 const logPath = join(opts.runDir, 'ramp.jsonl')
319 const first = steps[0]
320 if (!first) return fail('ramp', 'schedule has no step; traffic was not changed')
321 beginRamp(first.percent)
322 try {
323 const result = await runRamp(policy, {
324 now: () => Date.now(),
325 sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),
326 dwell: async (ms, percent) => {
327 const outcomes = await dwell(opts.token, config.baseUrl, ms, rollout.probe)
328 const loss = accountProbeLoss(outcomes, rollout.probe.maxLossRate)
329 const log = activeLog()
330 if (!log) throw new HarnessFault('probe-loss observation with no active log')
331 const at = Date.now()
332 await log.observation({
333 measurement: {
334 source: 'probe-loss',
335 percent,
336 count: loss.count,
337 rate: loss.rate,
338 total: loss.total,
339 responded: loss.responded,
340 serverErrors: loss.serverErrors,
341 timedOut: loss.timedOut,
342 shed: loss.shed,
343 aborted: loss.aborted,
344 network: loss.network,
345 maxLossRate: loss.maxLossRate,
346 exceeded: loss.exceeded,
347 ids: [...loss.ids],
348 },
349 query: { source: 'probe' },
350 from: at,
351 to: at,
352 raw: JSON.stringify({ ids: loss.ids }),
353 })
354 return { outcomes }
355 },
356 applySplit: async percent => {
357 const want = trafficSplit(percent, opts.version, target)
358 if (!isTrafficSplit(want)) return want.reason
359 if (sameSplit(expected, want.versions)) {
360 noteStepApplied(percent)
361 return null
362 }
363 try {
364 const applied = await applySplit(config.worker, `ramp ${opts.version} ${percent}%`, want, expected, { kind: 'step', percent })
365 if (applied.ok) {
366 expected = want.versions.map(share => ({ versionId: share.versionId, percentage: share.percentage }))
367 noteStepApplied(percent)
368 }
369 if (!applied.ok && applied.kind === 'state_conflict') return { kind: 'state_conflict', reason: applied.reason }
370 if (!applied.ok) return applied.reason
371 return null
372 } catch (error) {
373 if (error instanceof HarnessFault) throw error
374 return errorMessage(error)
375 }
376 },
377 onApplied: schedule.onApplied,
378 observe: (from, to, percent, sample) => observe(config, opts.version, target, from, queryTo(to), percent, sample),
379 onDecision: async (ref, decision, percent) => {
380 const log = activeLog()
381 if (!log) throw new HarnessFault('ramp decision with no active log')
382 log.addStep({ percent, burns: decision.burns })
383 await log.decision({
384 inputs: [ref],
385 rules: decision.rules,
386 outcome: decision.outcome,
387 reason: decision.reason,
388 })
389 if (decision.outcome !== 'advance') return
390 const index = steps.findIndex(step => step.percent === percent)
391 const next = steps[index + 1]
392 if (next) armStep(next.percent)
393 },
394 rollback: async () => {
395 const applied = await rollbackTo(config.worker, target, expected)
396 if (applied.ok) return null
397 if (applied.kind === 'state_conflict') return { kind: 'state_conflict', reason: applied.reason }
398 return applied.reason
399 },
400 })
401 const probeLoss = result.steps.map(step => probeLossRecord(step.percent, step.probeLoss))
402 for (const step of result.steps) {
403 opts.log(step.line)
404 if (step.forecast) opts.log(step.forecast.line)
405 await appendFile(logPath, JSON.stringify({ line: step.line, probe_loss: probeLossRecord(step.percent, step.probeLoss), forecast: step.forecast }) + '\n')
406 }
407 await writeJson(opts.runDir, 'ramp.json', {
408 state: result.state,
409 lines: result.lines,
410 steps: result.steps,
411 probeLoss,
412 rollbackError: result.rollbackError,
413 version: opts.version,
414 previous: target,
415 })
416 return rampStageResult(result, opts.version, target)
417 } catch (error) {
418 if (!(error instanceof HarnessFault)) throw error
419 let rolled = false
420 try {
421 const applied = await rollbackTo(config.worker, target, expected)
422 rolled = applied.ok
423 } catch (rollbackError) {
424 return fail('ramp', `${error.message}; rollback failed: ${errorMessage(rollbackError)}`, false, 'harness_fault')
425 }
426 return fail('ramp', error.message, rolled, 'harness_fault')
427 } finally {
428 endRamp()
429 }
Diagnostics4 git calls · 427 ms
CallStatusSizeTime0427 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=4
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=4
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2005.8 KiB421 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=421
git.response.bytes=5978
git.response.cache=MISS
git.response.duration_ms=421
git.fetch.wants=2afa00e9bf0198a917fd91ce7a332291fdcb5d44
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5978
git.pkt.payload.max=5957
git.sideband.band1.bytes=5956
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e335bcff4af485
Request
2c0fc563…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format