main
1// Prediction vs observation per ramp step (scripts/cd/forecast.ts, issue #73), and its wiring in runRamp.
2import assert from 'node:assert/strict'
3import { readFileSync } from 'node:fs'
4import { test } from 'node:test'
5import {
6 compareStep,
7 detectableErrorRate,
8 detectableWallMs,
9 mean,
10 observedBasis,
11 observeStep,
12 predictStep,
13 probeBasis,
14 variance,
15} from '../scripts/cd/forecast.ts'
16import { parsePolicy, type RolloutPolicy } from '../scripts/cd/policy.ts'
17import { runRamp, type Observation } from '../scripts/cd/ramp.ts'
18import type { VersionStats } from '../scripts/cd/telemetry.ts'
20function policy(steps: { percent: number; dwellMs: number }[]): RolloutPolicy {
21 const raw = JSON.parse(readFileSync(new URL('../scripts/cd/cd.policy.json', import.meta.url), 'utf8')) as Record<string, unknown>
22 raw.steps = steps
23 return parsePolicy(JSON.stringify(raw)).policy
24}
26function stats(over: Partial<VersionStats> = {}): VersionStats {
27 const row: VersionStats = { requests: 0, invocations: 0, exceptions: 0, errors: 0, fivexx: 0, errorLogs: 0, wallMs: [], cpuMs: [], fetchMs: [], v1: [], internal: [], ...over }
28 return { ...row, invocations: 'invocations' in over ? row.invocations : row.requests }
29}
31function sample(next: Partial<VersionStats>, previous: Partial<VersionStats>): Extract<Observation, { kind: 'sample' }> {
32 const newer = stats(next)
33 const older = stats(previous)
34 const aggregate = stats({ requests: newer.requests + older.requests, wallMs: [...newer.wallMs, ...older.wallMs] })
35 return { kind: 'sample', next: newer, previous: older, aggregate, landed: aggregate.invocations }
36}
38test('mean and sample variance', () => {
39 assert.equal(mean([]), null)
40 assert.equal(mean([1, 2, 3]), 2)
41 assert.equal(variance([5]), null)
42 assert.equal(variance([2, 4, 4, 4, 5, 5, 7, 9]), 32 / 7)
43})
45test('detectable effects: the rule of three at zero errors; z standard errors with a baseline; none without samples', () => {
46 assert.equal(detectableErrorRate(0, 100, 0, 3), null)
47 assert.equal(detectableErrorRate(300, 0, 0, 3), 0.01)
48 assert.equal(detectableErrorRate(300, 700, 0, 3), 0.01)
49 // p = 0.05, n = 1000/1000, z = 3: 3 * sqrt(0.0475 * 0.002) = 0.02924
50 assert.ok(Math.abs(Number(detectableErrorRate(1000, 1000, 0.05, 3)) - 0.029240) < 1e-5)
51 assert.equal(detectableWallMs(100, 100, null, 3), null)
52 assert.equal(detectableWallMs(100, 0, 400, 3), null)
53 // s = 20ms, n = 100/100, z = 3: 3 * 20 * sqrt(0.02) = 8.485
54 assert.ok(Math.abs(Number(detectableWallMs(100, 100, 400, 3)) - 8.4853) < 1e-3)
55})
57test('the first step is predicted from the probe rate; later steps from the step before', () => {
58 const p = policy([{ percent: 10, dwellMs: 40_000 }, { percent: 100, dwellMs: 40_000 }])
59 const first = probeBasis(p)
60 assert.equal(first.ratePerSecond, p.probe.rps * p.probe.calls.length)
61 const predicted = predictStep(first, 10, 40_000, 3)
62 // 25 rps x 2 calls x 40 s = 2000 requests, 10% new.
63 assert.deepEqual([predicted.newRequests, predicted.oldRequests, predicted.detectableWallMs], [200, 1800, null])
64 assert.equal(predicted.detectableErrorRate, 3 / 200)
65 const seen = sample({ requests: 200, wallMs: [90, 110] }, { requests: 1800, fivexx: 18, wallMs: [80, 120] })
66 const basis = observedBasis(1, 10, seen, 40_000)
67 assert.equal(basis.source, 'step 1 (10%)')
68 assert.equal(basis.ratePerSecond, 50)
69 assert.equal(basis.errorRate, 18 / 2000)
70 assert.equal(basis.wallVariance, variance([90, 110, 80, 120]))
71 assert.equal(basis.wallShare, 4 / 2000)
72})
74test('a step that sees the predicted traffic and variance has no drift; the line shows both', () => {
75 // One wall sample per hundred requests: the prediction counts wall samples, not requests, for the wall-time effect.
76 const basis = { source: 'step 1 (10%)', ratePerSecond: 50, errorRate: 0, wallVariance: 400, wallShare: 0.01 }
77 const predicted = predictStep(basis, 25, 40_000, 3)
78 assert.deepEqual([predicted.newRequests, predicted.oldRequests], [500, 1500])
79 const walls = (n: number) => Array.from({ length: n }, (_, i) => 100 + (i % 2) * 40)
80 const observed = observeStep(sample({ requests: 520, wallMs: walls(5) }, { requests: 1480, wallMs: walls(15) }), 3)
81 const check = compareStep({ step: 2, percent: 25, basis, predicted, observed: observed.resolution, wallDeltaMs: observed.wallDeltaMs })
82 assert.deepEqual(check.drift, [])
83 assert.ok(check.wallDeltaMs !== null && Math.abs(check.wallDeltaMs) < 5)
84 assert.match(check.line, /^forecast step=2 percent=25 predicted\(step 1 \(10%\)\) new=500 old=1500 detectable_error_rate=0\.0060 detectable_wall_ms=31\.0 \| observed new=520 old=1480 detectable_error_rate=0\.0058 detectable_wall_ms=\d+\.\d wall_delta_ms=-?\d+\.\d drift=none$/)
85})
87test('traffic far from the prediction, or a step that resolves far less than predicted, is drift', () => {
88 const basis = { source: 'step 1 (10%)', ratePerSecond: 50, errorRate: 0, wallVariance: 100, wallShare: 1 }
89 const predicted = predictStep(basis, 50, 40_000, 3)
90 const thin = observeStep(sample({ requests: 300, wallMs: [10, 400, 20, 900] }, { requests: 1000, wallMs: [15, 600, 30] }), 3)
91 const check = compareStep({ step: 3, percent: 50, basis, predicted, observed: thin.resolution, wallDeltaMs: thin.wallDeltaMs })
92 assert.deepEqual(check.drift.map(d => d.split(' ').slice(0, 2).join(' ')), ['new-version requests', 'detectable error', 'detectable wall'])
93 assert.match(check.line, / drift="new-version requests 300 vs predicted 1000; /)
94 // No basis: the observation is still logged, with nothing to drift from.
95 const none = compareStep({ step: 1, percent: 10, basis: null, predicted: null, observed: thin.resolution, wallDeltaMs: null })
96 assert.deepEqual(none.drift, [])
97 assert.match(none.line, /^forecast step=1 percent=10 predicted=none \| observed new=300 old=1000 .* drift=none$/)
98})
100test('runRamp records each step\'s forecast, predicting step n from step n-1', async () => {
101 let calls = 0
102 const result = await runRamp(policy([{ percent: 50, dwellMs: 1000 }, { percent: 100, dwellMs: 1000 }]), {
103 now: () => 0,
104 sleep: async () => {},
105 applySplit: async () => null,
106 observe: async () => {
107 calls += 1
108 if (calls === 1) return sample({ requests: 100, wallMs: [80, 90, 100, 110, 120] }, { requests: 100, wallMs: [80, 90, 100, 110, 120] })
109 return sample({ requests: 1000, wallMs: [80, 90, 100, 110, 120] }, { requests: 0 })
110 },
111 rollback: async () => null,
112 })
113 assert.equal(result.state.kind, 'complete')
114 const [first, second] = result.steps.map(step => {
115 assert.ok(step.forecast, `step ${step.percent} has a forecast`)
116 return step.forecast
117 })
118 // No dwell (no probe): the first step has nothing to predict from.
119 assert.equal(first.predicted, null)
120 assert.equal(first.observed.newRequests, 100)
121 // 200 requests in 1 s at step 1 predicts 200 new at 100%; 1000 is five times that.
122 assert.equal(second.basis, 'step 1 (50%)')
123 assert.ok(second.predicted)
124 assert.equal(second.predicted.newRequests, 200)
125 assert.deepEqual(second.drift, ['new-version requests 1000 vs predicted 200'])
126})
Diagnostics4 git calls · 229 ms
CallStatusSizeTime0229 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2002.4 KiB224 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=224 git.response.bytes=2470 git.response.cache=MISS git.response.duration_ms=224 git.fetch.wants=98222c7ede50fdb71b5cb8f7037716f16c3fd8b4 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2470 git.pkt.payload.max=2449 git.sideband.band1.bytes=2448 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2fa74ad4890c0
- Request
- 355b49c4…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format