main
ply4 git calls34 ms44 KiB3 cachedprotocol v2
194 lines9.7 KiBRaw
1// Prediction vs observation for each ramp step (issue #73). Before a step, the previous step's measurements (or, for
2// the first step, the policy's probe rate) predict how many requests each version will serve during the dwell and the
3// smallest effect the step can then detect: an error-rate increase and a wall-time shift at the policy's z. After the
4// step, the same quantities are computed from what it saw. A step whose traffic or resolution is more than `tolerance`
5// times off the prediction is flagged as drift: the policy's dwell, z, and sample floors were set for a different
6// distribution than the one the ramp is running on. Drift is logged next to the step line; it is never a gate.
7//
8// Detectable effects (two-sided normal approximation, the same z the routed-share band uses, policy.split.z):
9// error rate max(3 / n_new, z * sqrt(p (1 - p) (1/n_new + 1/n_old))) where p is the baseline error rate. 3/n is
10// the rule of three: with no error in n requests, rates above 3/n are excluded at about 95%.
11// wall time z * sqrt(s^2 (1/n_new + 1/n_old)) ms, s^2 the measured wall-time variance of the basis step and n the
12// wall-time samples (requests x the basis step's samples per request).
13import type { RolloutPolicy } from './policy.ts'
14import type { VersionStats } from './telemetry.ts'
16/** What a prediction is made from. */
17export type ForecastBasis = {
18 /** Where the numbers came from: `policy probe` or `step <n> (<percent>%)`. */
19 readonly source: string
20 /** External requests per second across the Worker. */
21 readonly ratePerSecond: number
22 /** Baseline bad-request rate (5xx and exceptions over requests). */
23 readonly errorRate: number
24 /** Wall-time variance in ms^2, or null when not measured. */
25 readonly wallVariance: number | null
26 /** Wall-time samples per request (not every request carries one). 1 when not measured. */
27 readonly wallShare: number
30/** The predicted or observed counts and resolution of one step. */
31export type StepResolution = {
32 readonly newRequests: number
33 readonly oldRequests: number
34 /** Smallest error-rate increase the step resolves. Null with no new-version request. */
35 readonly detectableErrorRate: number | null
36 /** Smallest wall-time shift in ms the step resolves. Null without a variance or without both versions. */
37 readonly detectableWallMs: number | null
40export type ForecastCheck = {
41 readonly percent: number
42 readonly predicted: StepResolution | null
43 readonly basis: string | null
44 readonly observed: StepResolution
45 /** Observed wall-time mean difference new - old in ms, when both sides have samples. */
46 readonly wallDeltaMs: number | null
47 readonly drift: readonly string[]
48 readonly line: string
51/** Ratio beyond which a prediction has drifted. PRIOR (uncalibrated): a step that saw half or double the traffic it was sized for. */
52export const FORECAST_TOLERANCE = 2
54/** Rule of three: zero events in n bounds the rate at about 3/n (95%). */
55const RULE_OF_THREE = 3
56const MS_PER_SECOND = 1000
58export function mean(values: readonly number[]): number | null {
59 const finite = values.filter(value => Number.isFinite(value))
60 if (finite.length === 0) return null
61 return finite.reduce((sum, value) => sum + value, 0) / finite.length
64/** Sample variance (n - 1). Null below two samples. */
65export function variance(values: readonly number[]): number | null {
66 const finite = values.filter(value => Number.isFinite(value))
67 if (finite.length < 2) return null
68 const m = finite.reduce((sum, value) => sum + value, 0) / finite.length
69 return finite.reduce((sum, value) => sum + (value - m) ** 2, 0) / (finite.length - 1)
72export function detectableErrorRate(newRequests: number, oldRequests: number, baseline: number, z: number): number | null {
73 if (!(newRequests > 0)) return null
74 const floor = RULE_OF_THREE / newRequests
75 if (!(oldRequests > 0)) return floor
76 const p = Math.min(Math.max(baseline, 0), 1)
77 return Math.max(floor, z * Math.sqrt(p * (1 - p) * (1 / newRequests + 1 / oldRequests)))
80export function detectableWallMs(newRequests: number, oldRequests: number, wallVariance: number | null, z: number): number | null {
81 if (wallVariance === null || !(newRequests > 0) || !(oldRequests > 0)) return null
82 return z * Math.sqrt(wallVariance * (1 / newRequests + 1 / oldRequests))
85/** The first step's basis: the probe alone (rps x calls). Real traffic adds to it, so this is a floor. */
86export function probeBasis(policy: RolloutPolicy): ForecastBasis {
87 return { source: 'policy probe', ratePerSecond: policy.probe.rps * policy.probe.calls.length, errorRate: 0, wallVariance: null, wallShare: 1 }
90function bad(stats: VersionStats): number {
91 return stats.fivexx + stats.exceptions
94function served(stats: VersionStats): number {
95 return stats.invocations > 0 ? stats.invocations : stats.requests
98/** The next step's basis: this step's measured rate, error rate and wall-time variance (both versions pooled). */
99export function observedBasis(step: number, percent: number, sample: { readonly next: VersionStats; readonly previous: VersionStats; readonly aggregate: VersionStats }, dwellMs: number): ForecastBasis {
100 const total = served(sample.aggregate)
101 const errors = bad(sample.next) + bad(sample.previous)
102 const versions = served(sample.next) + served(sample.previous)
103 return {
104 source: `step ${step} (${percent}%)`,
105 ratePerSecond: dwellMs > 0 ? total / (dwellMs / MS_PER_SECOND) : 0,
106 errorRate: versions > 0 ? errors / versions : 0,
107 wallVariance: variance([...sample.next.wallMs, ...sample.previous.wallMs]),
108 wallShare: versions > 0 ? (sample.next.wallMs.length + sample.previous.wallMs.length) / versions : 1,
109 }
112export function predictStep(basis: ForecastBasis, percent: number, dwellMs: number, z: number): StepResolution {
113 const total = basis.ratePerSecond * (dwellMs / MS_PER_SECOND)
114 const share = Math.min(Math.max(percent / 100, 0), 1)
115 const newRequests = Math.round(total * share)
116 const oldRequests = Math.round(total * (1 - share))
117 return {
118 newRequests,
119 oldRequests,
120 detectableErrorRate: detectableErrorRate(newRequests, oldRequests, basis.errorRate, z),
121 detectableWallMs: detectableWallMs(newRequests * basis.wallShare, oldRequests * basis.wallShare, basis.wallVariance, z),
122 }
125export function observeStep(sample: { readonly next: VersionStats; readonly previous: VersionStats }, z: number): { readonly resolution: StepResolution; readonly wallDeltaMs: number | null } {
126 const newRequests = served(sample.next)
127 const oldRequests = served(sample.previous)
128 const baseline = oldRequests > 0 ? bad(sample.previous) / oldRequests : 0
129 const nextMean = mean(sample.next.wallMs)
130 const previousMean = mean(sample.previous.wallMs)
131 return {
132 resolution: {
133 newRequests,
134 oldRequests,
135 detectableErrorRate: detectableErrorRate(newRequests, oldRequests, baseline, z),
136 detectableWallMs: detectableWallMs(sample.next.wallMs.length, sample.previous.wallMs.length, variance([...sample.next.wallMs, ...sample.previous.wallMs]), z),
137 },
138 wallDeltaMs: nextMean === null || previousMean === null ? null : nextMean - previousMean,
139 }
142/** `observed` is more than `tolerance` times off `predicted`, either way. Zero against non-zero is off. */
143function off(predicted: number, observed: number, tolerance: number): boolean {
144 if (predicted === 0 && observed === 0) return false
145 if (predicted === 0 || observed === 0) return true
146 const ratio = observed / predicted
147 return ratio > tolerance || ratio < 1 / tolerance
150function num(value: number | null, digits = 0): string {
151 if (value === null) return '-'
152 return digits === 0 ? String(Math.round(value)) : value.toFixed(digits)
155function resolutionText(r: StepResolution): string {
156 return `new=${r.newRequests} old=${r.oldRequests} detectable_error_rate=${num(r.detectableErrorRate, 4)} detectable_wall_ms=${num(r.detectableWallMs, 1)}`
159/** Prediction next to observation, and what drifted. `predicted` is null when there was no basis. */
160export function compareStep(input: {
161 readonly step: number
162 readonly percent: number
163 readonly basis: ForecastBasis | null
164 readonly predicted: StepResolution | null
165 readonly observed: StepResolution
166 readonly wallDeltaMs: number | null
167}, tolerance: number = FORECAST_TOLERANCE): ForecastCheck {
168 const drift: string[] = []
169 const p = input.predicted
170 const o = input.observed
171 if (p !== null) {
172 if (off(p.newRequests, o.newRequests, tolerance)) drift.push(`new-version requests ${o.newRequests} vs predicted ${p.newRequests}`)
173 if (off(p.oldRequests, o.oldRequests, tolerance)) drift.push(`old-version requests ${o.oldRequests} vs predicted ${p.oldRequests}`)
174 if (p.detectableErrorRate !== null && o.detectableErrorRate !== null && o.detectableErrorRate > p.detectableErrorRate * tolerance) {
175 drift.push(`detectable error rate ${num(o.detectableErrorRate, 4)} vs predicted ${num(p.detectableErrorRate, 4)}`)
176 }
177 if (p.detectableWallMs !== null && o.detectableWallMs !== null && o.detectableWallMs > p.detectableWallMs * tolerance) {
178 drift.push(`detectable wall shift ${num(o.detectableWallMs, 1)}ms vs predicted ${num(p.detectableWallMs, 1)}ms`)
179 }
180 }
181 const source = input.basis === null ? null : input.basis.source
182 const predicted = p === null ? 'predicted=none' : `predicted(${source === null ? '-' : source}) ${resolutionText(p)}`
183 const seen = `observed ${resolutionText(o)} wall_delta_ms=${num(input.wallDeltaMs, 1)}`
184 const verdict = drift.length === 0 ? 'drift=none' : `drift="${drift.join('; ')}"`
185 return {
186 percent: input.percent,
187 predicted: p,
188 basis: source,
189 observed: o,
190 wallDeltaMs: input.wallDeltaMs,
191 drift,
192 line: `forecast step=${input.step} percent=${input.percent} ${predicted} | ${seen} ${verdict}`,
193 }
Diagnostics4 git calls · 34 ms
CallStatusSizeTime034 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch2003.0 KiB · hit28 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=27
git.response.bytes=3086
git.response.cache=HIT
git.response.duration_ms=28
git.fetch.wants=95151568689936031b8da18d28ba80d0d09eba6e
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3086
git.pkt.payload.max=3065
git.sideband.band1.bytes=3064
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e2e44a58412947
Request
c1eb7f5e…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format