main
1// The rate limiter's silent-degradation gate (docs/rate-limit.md "CD gate"). The limiter fails soft by design: a broken
2// read admits (fail_open), a flush that cannot reach the counter is dropped after 5 s, and a missing counter binding
3// sends every request to `legacy` (59926117 / e61a5742, 2026-11-06: ctx.exports.RateCounter was a function, so batched
4// fell back to legacy and nothing alerted). Each of those is a 200 to the client, so the smoke and the error gates pass.
5// This gate reads the version's own `ply.rate.check` / `ply.rate.flush` spans and fails it when:
6//
7// mode a default-mode check (not chosen by a load request's x-ply-rate-mode) ran a limiter other than the policy's
8// fail_open warm default-mode checks failed open above maxFailOpenRate. A check the Worker stamps `rate.cold` is an
9// expected fail-open and is counted on its own line, out of the warm share: `isolate_start` (the isolate's
10// first read is in flight) and `memo_expired` (an idle isolate's document aged past MEMO_FALLBACK_MS;
11// ply-web#156). A memo_expired check whose own refresh then failed is put back as a warm failure.
12// dropped flushes dropped (deadline or overflow) with no injected fault, above maxDroppedFlushes
13//
14// The gate sends its own pinned probes first (the viewer redirect route: rate-checked, no git work), carrying the load
15// token and a run-unique client label but no mode header, so the version has default-mode checks even at 0% and the
16// probes never spend the operator's address quota (the visual smoke shares it).
17import { atIndex, dig, recordGet } from '../../src/absent.ts'
18import { unwrapOr } from '../../src/option.ts'
19import type { ExpectedCold } from '../../src/rate/admit.ts'
20import type { RateGatePolicy } from './policy.ts'
21import type { CountRow } from './o11y.ts'
22import type { Filter } from './telemetry.ts'
24/** Default-mode checks of one expected cold cause, and how many of them failed open. */
25export type ColdTally = { readonly checks: number; readonly failOpen: number }
27export const EXPECTED_COLD: readonly ExpectedCold[] = ['isolate_start', 'memo_expired']
29export type RateCounts = {
30 /** Default-mode checks by `rate.mode`. */
31 readonly modes: Readonly<Record<string, number>>
32 readonly defaultChecks: number
33 readonly failOpen: number
34 /** Default-mode checks stamped `rate.cold`, by cause: expected fail-opens, out of the warm share. */
35 readonly cold: Readonly<Record<ExpectedCold, ColdTally>>
36 /** `ply.rate.refresh` started by a memo_expired check, by outcome. A failed one is a warm read-path failure. */
37 readonly expiredRefresh: { readonly ok: number; readonly failed: number }
38 /** Checks whose mode a load request chose (excluded from every judgement). */
39 readonly headerChecks: number
40 readonly overflowDrops: number
41 readonly flushes: number
42 readonly droppedFlushes: number
43 readonly faultedDrops: number
44}
46/** The warm share's inputs: expected cold checks out, memo_expired ones whose refresh failed back in. Pure. */
47export function warmShare(counts: RateCounts): { readonly warm: number; readonly warmOpen: number; readonly back: number } {
48 const coldChecks = EXPECTED_COLD.reduce((n, k) => n + counts.cold[k].checks, 0)
49 const coldOpen = EXPECTED_COLD.reduce((n, k) => n + counts.cold[k].failOpen, 0)
50 const back = Math.min(counts.expiredRefresh.failed, counts.cold.memo_expired.failOpen)
51 return { warm: counts.defaultChecks - coldChecks + back, warmOpen: counts.failOpen - coldOpen + back, back }
52}
54export type RateVerdict =
55 | { readonly kind: 'pass'; readonly line: string; readonly counts: RateCounts }
56 | { readonly kind: 'worker'; readonly line: string; readonly counts: RateCounts; readonly failures: readonly string[] }
57 | { readonly kind: 'harness_fault'; readonly line: string; readonly counts: RateCounts | null }
59const PREFIX = 'rate:'
61function pct(n: number): string {
62 return `${(n * 100).toFixed(2)}%`
63}
65/** Judge one window's counts. Pure. A mode or drop failure needs no minimum; the fail_open share needs minChecks. */
66export function judgeRate(counts: RateCounts, policy: RateGatePolicy): RateVerdict {
67 const failures: string[] = []
68 const wrong = Object.entries(counts.modes).filter(([mode, n]) => mode !== policy.expectMode && n > 0)
69 if (wrong.length > 0) {
70 const seen = wrong.map(([mode, n]) => `${mode} ${n}`).join(', ')
71 failures.push(`mode: ${seen} of ${counts.defaultChecks} default-mode checks ran a limiter other than ${policy.expectMode} (silent fallback; see 59926117)`)
72 }
73 const dropped = counts.droppedFlushes - counts.faultedDrops + counts.overflowDrops
74 if (dropped > policy.maxDroppedFlushes) {
75 failures.push(`dropped: ${dropped} batches lost without an injected fault (deadline ${counts.droppedFlushes - counts.faultedDrops}, overflow ${counts.overflowDrops}) of ${counts.flushes} flushes > ${policy.maxDroppedFlushes}`)
76 }
77 const { warm, warmOpen, back } = warmShare(counts)
78 const share = warm > 0 ? warmOpen / warm : 0
79 const { isolate_start: start, memo_expired: expired } = counts.cold
80 const expected = `expected fail_open isolate_start ${start.failOpen}/${start.checks}, memo_expired ${expired.failOpen}/${expired.checks} (refresh ok ${counts.expiredRefresh.ok}, failed ${counts.expiredRefresh.failed}${back > 0 ? `, ${back} counted warm` : ''})`
81 const summary = `default ${counts.defaultChecks}, fail_open warm ${warmOpen}/${warm} = ${pct(share)}; ${expected}; flushes ${counts.flushes} dropped ${dropped}, header-mode ${counts.headerChecks}`
82 if (warm >= policy.minChecks && share > policy.maxFailOpenRate) {
83 failures.push(`fail_open: ${warmOpen}/${warm} warm default-mode checks failed open = ${pct(share)} > ${pct(policy.maxFailOpenRate)}`)
84 }
85 if (failures.length > 0) return { kind: 'worker', line: `${PREFIX} FAIL ${failures.join('; ')} [${summary}]`, counts, failures }
86 if (warm < policy.minChecks) {
87 return { kind: 'harness_fault', line: `${PREFIX} ${warm} warm default-mode checks < minChecks ${policy.minChecks}: the fail_open share cannot be judged [${summary}]`, counts }
88 }
89 return { kind: 'pass', line: `${PREFIX} mode ${policy.expectMode} on every default check; ${summary}`, counts }
90}
92// ---- counting: from aggregate rows (production) or raw span events (tests, and a dump) ----
94function sum(rows: readonly CountRow[], match: (g: Readonly<Record<string, string>>) => boolean = () => true): number {
95 let n = 0
96 for (const row of rows) if (match(row.groups)) n += row.count
97 return n
98}
100/** The aggregate rows the gate's queries return, by query. */
101export type RateRows = {
102 /** ply.rate.check grouped by rate.mode, rate.decision. */
103 readonly checks: readonly CountRow[]
104 /** The same, only rate.mode_source = header. */
105 readonly header: readonly CountRow[]
106 /** ply.rate.check with rate.cold present, grouped by rate.cold, rate.mode_source, rate.decision. */
107 readonly cold: readonly CountRow[]
108 /** ply.rate.refresh with rate.refresh.cold = memo_expired, grouped by rate.refresh.read. */
109 readonly expiredRefresh: readonly CountRow[]
110 /** ply.rate.check with rate.batch.dropped present. */
111 readonly overflow: readonly CountRow[]
112 /** ply.rate.flush grouped by rate.batch.outcome. */
113 readonly flushes: readonly CountRow[]
114 /** ply.rate.flush, outcome dropped, rate.fault present. */
115 readonly faulted: readonly CountRow[]
116}
118function bump(counts: Record<string, number>, key: string, by: number): void {
119 const next = unwrapOr(recordGet(counts, key), 0) + by
120 if (next === 0) delete counts[key]
121 else counts[key] = next
122}
124function isExpectedCold(value: unknown): value is ExpectedCold {
125 return EXPECTED_COLD.some(k => k === value)
126}
128function tallyCold(rows: readonly CountRow[]): Record<ExpectedCold, ColdTally> {
129 const tally = (kind: ExpectedCold): ColdTally => ({
130 checks: sum(rows, g => g['rate.cold'] === kind && g['rate.mode_source'] !== 'header'),
131 failOpen: sum(rows, g => g['rate.cold'] === kind && g['rate.mode_source'] !== 'header' && g['rate.decision'] === 'fail_open'),
132 })
133 return { isolate_start: tally('isolate_start'), memo_expired: tally('memo_expired') }
134}
136export function countsFromRows(rows: RateRows): RateCounts {
137 const modes: Record<string, number> = {}
138 for (const row of rows.checks) bump(modes, unwrapOr(recordGet(row.groups, 'rate.mode'), '?'), row.count)
139 for (const row of rows.header) bump(modes, unwrapOr(recordGet(row.groups, 'rate.mode'), '?'), -row.count)
140 const failOpen = (g: Readonly<Record<string, string>>): boolean => g['rate.decision'] === 'fail_open'
141 const refreshOk = sum(rows.expiredRefresh, g => g['rate.refresh.read'] === 'ok')
142 return {
143 modes,
144 defaultChecks: sum(rows.checks) - sum(rows.header),
145 failOpen: sum(rows.checks, failOpen) - sum(rows.header, failOpen),
146 cold: tallyCold(rows.cold),
147 expiredRefresh: { ok: refreshOk, failed: sum(rows.expiredRefresh) - refreshOk },
148 headerChecks: sum(rows.header),
149 overflowDrops: sum(rows.overflow),
150 flushes: sum(rows.flushes),
151 droppedFlushes: sum(rows.flushes, g => g['rate.batch.outcome'] === 'dropped'),
152 faultedDrops: sum(rows.faulted),
153 }
154}
156/** A span attribute under `source`, by dotted path. Absent is `null`. */
157function attr(event: unknown, key: string): unknown {
158 return unwrapOr(dig(event, ['source', ...key.split('.')]), null)
159}
161function spanName(event: unknown): string {
162 const meta = dig(event, ['$metadata', 'spanName'])
163 if (meta.tag === 'some' && typeof meta.value === 'string') return meta.value
164 const name = attr(event, 'name')
165 return typeof name === 'string' ? name : ''
166}
168/**
169 * Counts from raw span events (Workers Observability `events` rows of `$metadata.type = span`). A check with no
170 * `rate.mode_source` comes from a version that predates the attribute: it is judged as default-mode, so a load request's
171 * header-chosen arm on such a version would count against it (the gate only runs on versions that stamp it).
172 */
173export function countsFromSpans(events: readonly unknown[], version: string): RateCounts {
174 const modes: Record<string, number> = {}
175 const cold: Record<ExpectedCold, { checks: number; failOpen: number }> = { isolate_start: { checks: 0, failOpen: 0 }, memo_expired: { checks: 0, failOpen: 0 } }
176 const expiredRefresh = { ok: 0, failed: 0 }
177 let defaultChecks = 0, failOpen = 0, headerChecks = 0, overflowDrops = 0, flushes = 0, droppedFlushes = 0, faultedDrops = 0
178 for (const event of events) {
179 if (attr(event, 'ply.script_version') !== version) continue
180 const name = spanName(event)
181 if (name === 'ply.rate.check') {
182 if (attr(event, 'rate.batch.dropped') !== null) overflowDrops++
183 if (attr(event, 'rate.mode_source') === 'header') { headerChecks++; continue }
184 const mode = attr(event, 'rate.mode')
185 bump(modes, mode === null ? '?' : String(mode), 1)
186 defaultChecks++
187 const open = attr(event, 'rate.decision') === 'fail_open'
188 if (open) failOpen++
189 const kind = attr(event, 'rate.cold')
190 if (isExpectedCold(kind)) {
191 cold[kind].checks++
192 if (open) cold[kind].failOpen++
193 }
194 } else if (name === 'ply.rate.refresh') {
195 if (attr(event, 'rate.refresh.cold') !== 'memo_expired') continue
196 if (attr(event, 'rate.refresh.read') === 'ok') expiredRefresh.ok++
197 else expiredRefresh.failed++
198 } else if (name === 'ply.rate.flush') {
199 flushes++
200 if (attr(event, 'rate.batch.outcome') === 'dropped') {
201 droppedFlushes++
202 if (attr(event, 'rate.fault') !== null) faultedDrops++
203 }
204 }
205 }
206 return { modes, defaultChecks, failOpen, cold, expiredRefresh, headerChecks, overflowDrops, flushes, droppedFlushes, faultedDrops }
207}
209// ---- the queries ----
211export type CountQuery = (filters: readonly Filter[], groupBys: readonly string[]) => Promise<CountRow[]>
213function eq(key: string, value: string): Filter {
214 return { key, operation: 'eq', type: 'string', value }
215}
217function exists(key: string): Filter {
218 return { key, operation: 'exists', type: 'string', value: '' }
219}
221/** The seven aggregate queries, scoped to the Worker and to the version's own `ply.script_version` stamp. */
222export async function queryRateRows(worker: string, version: string, count: CountQuery): Promise<RateRows> {
223 const base = [eq('$metadata.service', worker), eq('ply.script_version', version)]
224 const check = [...base, eq('$metadata.spanName', 'ply.rate.check')]
225 const flush = [...base, eq('$metadata.spanName', 'ply.rate.flush')]
226 const refresh = [...base, eq('$metadata.spanName', 'ply.rate.refresh')]
227 const [checks, header, cold, expiredRefresh, overflow, flushes, faulted] = await Promise.all([
228 count(check, ['rate.mode', 'rate.decision']),
229 count([...check, eq('rate.mode_source', 'header')], ['rate.mode', 'rate.decision']),
230 count([...check, exists('rate.cold')], ['rate.cold', 'rate.mode_source', 'rate.decision']),
231 count([...refresh, eq('rate.refresh.cold', 'memo_expired')], ['rate.refresh.read']),
232 count([...check, exists('rate.batch.dropped')], []),
233 count(flush, ['rate.batch.outcome']),
234 count([...flush, eq('rate.batch.outcome', 'dropped'), exists('rate.fault')], []),
235 ])
236 return { checks, header, cold, expiredRefresh, overflow, flushes, faulted }
237}
239// ---- the probes ----
241export type ProbeIo = {
242 readonly fetch: (url: string, init: { headers: Record<string, string>; redirect: 'manual'; signal: AbortSignal }) => Promise<{ status: number; headers: { get(name: string): string | null }; body: unknown }>
243 readonly sleep: (ms: number) => Promise<void>
244}
246export type ProbeResult = { readonly sent: number; readonly answered: number; readonly onVersion: number; readonly statuses: Readonly<Record<string, number>> }
248/**
249 * `policy.probes` sequential GETs of `policy.probePath`, pinned to `version`, as load client `label` (the load token
250 * keeps them off the operator's address quota). No x-ply-rate-mode: they run the version's default limiter.
251 */
252export async function sendRateProbes(input: { readonly baseUrl: string; readonly worker: string; readonly version: string; readonly loadToken: string; readonly label: string; readonly policy: RateGatePolicy }, io: ProbeIo): Promise<ProbeResult> {
253 const statuses: Record<string, number> = {}
254 let answered = 0
255 let onVersion = 0
256 for (let i = 0; i < input.policy.probes; i++) {
257 if (i > 0) await io.sleep(input.policy.probeGapMs)
258 let status = 'error'
259 try {
260 const res = await io.fetch(`${input.baseUrl}${input.policy.probePath}`, {
261 headers: {
262 'cloudflare-workers-version-overrides': `${input.worker}="${input.version}"`,
263 'x-ply-load-token': input.loadToken,
264 'x-ply-rate-client': input.label,
265 'user-agent': 'ply-cd-rate-gate/1',
266 },
267 redirect: 'manual',
268 signal: AbortSignal.timeout(10_000),
269 })
270 const cancel = dig(res, ['body', 'cancel'])
271 if (cancel.tag === 'some' && typeof cancel.value === 'function') await Promise.resolve((cancel.value as () => Promise<void>).call(res.body)).catch(() => {})
272 status = String(res.status)
273 answered++
274 if (res.headers.get('x-ply-version') === input.version) onVersion++
275 } catch {
276 status = 'error'
277 }
278 bump(statuses, status, 1)
279 }
280 return { sent: input.policy.probes, answered, onVersion, statuses }
281}
283export type RatePoll = {
284 readonly query: () => Promise<RateCounts>
285 readonly sleep: (ms: number) => Promise<void>
286 readonly now: () => number
287 readonly pollsMs: readonly number[]
288 readonly waitMs: number
289}
291/**
292 * Query and judge until the verdict is not a harness fault (enough warm checks ingested) or `waitMs` has passed. A
293 * mode or drop failure is final at once: more data cannot undo a check that already ran the wrong limiter.
294 */
295export async function pollRate(policy: RateGatePolicy, poll: RatePoll): Promise<RateVerdict> {
296 const started = poll.now()
297 for (let attempt = 0; ; attempt++) {
298 const verdict = judgeRate(await poll.query(), policy)
299 const waited = poll.now() - started
300 if (verdict.kind !== 'harness_fault' || waited >= poll.waitMs) return verdict
301 const pause = unwrapOr(atIndex(poll.pollsMs, Math.min(attempt, poll.pollsMs.length - 1)), poll.waitMs)
302 await poll.sleep(Math.min(pause, poll.waitMs - waited))
303 }
304}
Diagnostics4 git calls · 628 ms
CallStatusSizeTime0628 ms
discover200152 B4 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit4 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=4 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=4 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit5 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=5 git.response.bytes=41349 git.response.cache=HIT git.response.duration_ms=5 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=41349 git.pkt.payload.max=41254 git.sideband.band1.bytes=41253 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=61 git.pack.bytes=41253 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c git.pack.full=61Open pack map
fetch2005.1 KiB619 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=619 git.response.bytes=5216 git.response.cache=MISS git.response.duration_ms=619 git.fetch.wants=f7fadf8fde9e9d1f7782c2a8bf58863eb04118b6 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5216 git.pkt.payload.max=5195 git.sideband.band1.bytes=5194 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
- Ref
- refs/heads/main
- Commit
- 526b305930b8ef382f2d7a1f9c0238730a585921
- cf-ray
- a5e2eab00e814bc2
- Request
- f5d7529c…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format