main
ply4 git calls270 ms55 KiB2 cachedprotocol v2
977 lines52 KiBRaw
1// The batched rate limiter (docs/rate-limit.md): the counter's SQL on real SQLite (node:sqlite), the batcher under a
2// virtual clock, the cached document, and the request-path decision end to end with a fake ctx.
3import { test } from 'node:test';
4import assert from 'node:assert/strict';
5import { DatabaseSync } from 'node:sqlite';
6import { spawn } from 'node:child_process';
7import { createServer } from 'node:net';
8import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
9import { tmpdir } from 'node:os';
10import { join } from 'node:path';
11import { fileURLToPath } from 'node:url';
12import { copyCfBundle } from './cf-bundle.js';
13import { CounterCore, InvalidBatch, blockedUntil, firstLiveSlot, type ApplyRequest, type ApplyResult, type SqlExec, type SqlValue } from '../src/rate/core.ts';
14import { Batcher, defaultTuning, type BatcherDeps, type FlushReport } from '../src/rate/batcher.ts';
15import { SLOT_MS, WINDOW_MS, DOC_PATH, DOC_URL, KEY_HEX, MEMO_FALLBACK_MS, MEMO_REFRESH_ABANDON_MS, MEMO_REFRESH_MS, PRUNE_BATCH_ROWS, DEDUPE_RETENTION_MS } from '../src/rate/config.ts';
16import { indexDoc, serveDoc } from '../src/rate/doc.ts';
17import type { RefreshTrigger } from '../src/rate/admit.ts';
19const COLD_START: RefreshTrigger = { kind: 'cold', cause: { kind: 'isolate_start' } };
20const STALE: RefreshTrigger = { kind: 'stale' };
21import { counterNamespace } from '../src/rate/stub.ts';
22import { admitBatched, coldCause, parseFault, rateIdentity, readDoc, readMemo, refreshMemo, resetRateState, LOAD_CLIENT_HEADER, LOAD_MODE_HEADER, LOAD_TOKEN_HEADER } from '../src/rate/admit.ts';
24function sqlite(): SqlExec {
25 const db = new DatabaseSync(':memory:');
26 return (q: string, ...b: SqlValue[]) => {
27 const stmt = db.prepare(q);
28 if (/^\s*select/i.test(q)) {
29 const rows = stmt.all(...b) as Record<string, SqlValue>[];
30 return { toArray: () => rows, rowsWritten: 0 };
31 }
32 const r = stmt.run(...b);
33 return { toArray: () => [], rowsWritten: Number(r.changes) };
34 };
37const K1 = 'aaaaaaaaaaaaaaaa';
38const K2 = 'bbbbbbbbbbbbbbbb';
39const LIMITS: readonly [number, number] = [60, 20];
41function req(id: string, entries: ApplyRequest['entries'], attempt = 1): ApplyRequest {
42 return { id, attempt, limits: LIMITS, entries, fault: null };
45test('core: counts, and an id applies once however often it is replayed', () => {
46 const core = new CounterCore(sqlite());
47 const t = 1_000_000_000_000;
48 const first = core.apply(req('iso:1', [[K1, 0, 30]]), t);
49 assert.equal(first.replay, false);
50 assert.ok(first.rowsWritten >= 2);
51 for (let i = 0; i < 5; i++) {
52 const again = core.apply(req('iso:1', [[K1, 0, 30]], i + 2), t + i);
53 assert.equal(again.replay, true);
54 }
55 // 30 counted once: 29 more stays under 60.
56 assert.equal(core.apply(req('iso:2', [[K1, 0, 29]]), t + 10).newBlocks, 0);
57 assert.equal(core.doc(t + 10).entries.length, 0);
58 const crossing = core.apply(req('iso:3', [[K1, 0, 1]]), t + 20);
59 assert.equal(crossing.newBlocks, 1);
60 const doc = core.doc(t + 20);
61 assert.deepEqual(doc.entries.map(e => [e[0], e[1], e[3]]), [[K1, 0, t + 20]]);
62 assert.equal(crossing.over[0].peak, 60);
63});
65test('core: page and pack buckets are separate, and keys are separate', () => {
66 const core = new CounterCore(sqlite());
67 const t = 1_000_000_000_000;
68 core.apply(req('a:1', [[K1, 1, 20], [K2, 0, 20]]), t);
69 const doc = core.doc(t);
70 assert.deepEqual(doc.entries.map(e => `${e[0]}:${e[1]}`), [`${K1}:1`]);
71});
73test('core: window edges — a hit counts until its slot end + 60 s, then the block lifts', () => {
74 const core = new CounterCore(sqlite());
75 const slotStart = Math.floor(1_000_000_000_000 / SLOT_MS) * SLOT_MS;
76 const t = slotStart + 3_000;
77 core.apply(req('w:1', [[K1, 0, 60]]), t);
78 const until = core.doc(t).entries[0][2];
79 assert.equal(until, slotStart + SLOT_MS + WINDOW_MS);
80 assert.equal(core.doc(until - 1).entries.length, 1);
81 assert.equal(core.doc(until).entries.length, 0);
82 // At `until` the slot is out of the window: one more hit is 1, not 61.
83 const after = core.apply(req('w:2', [[K1, 0, 1]]), until);
84 assert.equal(after.newBlocks, 0);
85 assert.equal(firstLiveSlot(until), Math.floor(slotStart / SLOT_MS) + 1);
86});
88test('core: blockedUntil drops oldest slots until the sum is under the limit', () => {
89 const slots = [{ slot: 10, n: 30 }, { slot: 11, n: 20 }, { slot: 12, n: 20 }];
90 assert.equal(blockedUntil(slots, 70, 60), 11 * SLOT_MS + WINDOW_MS);
91 assert.equal(blockedUntil(slots, 70, 30), 12 * SLOT_MS + WINDOW_MS);
92});
94test('core: overshoot is peak − limit; until moves out as admitted hits keep coming', () => {
95 const core = new CounterCore(sqlite());
96 const t = Math.floor(1_000_000_000_000 / SLOT_MS) * SLOT_MS;
97 core.apply(req('o:1', [[K1, 0, 30]]), t);
98 const crossing = core.apply(req('o:2', [[K1, 0, 30]]), t + SLOT_MS);
99 assert.equal(crossing.newBlocks, 1);
100 assert.equal(core.doc(t + SLOT_MS).entries[0][2], t + SLOT_MS + WINDOW_MS, 'dropping the first slot is enough');
101 const g1 = core.gen();
102 const r = core.apply(req('o:3', [[K1, 0, 31]]), t + 2 * SLOT_MS);
103 assert.equal(r.newBlocks, 0);
104 assert.equal(r.over[0].peak, 91);
105 assert.equal(r.over[0].crossed, t + SLOT_MS);
106 assert.equal(core.doc(t + 2 * SLOT_MS).entries[0][2], t + 2 * SLOT_MS + WINDOW_MS, 'now the first two slots must go');
107 assert.ok(core.gen() > g1, 'a moved until bumps the generation');
108});
110test('core: pruning removes old ids, slots and ended blocks', () => {
111 const core = new CounterCore(sqlite());
112 const t = 1_000_000_000_000;
113 core.apply(req('p:1', [[K1, 0, 60]]), t);
114 core.apply(req('p:2', [[K2, 0, 1]]), t + 11 * 60_000);
115 assert.deepEqual(core.sizes(), { hits: 1, batches: 1, blocked: 0 });
116 // p:1 is gone from the dedupe table after retention: documented bound, far beyond the 5 s client deadline.
117});
119test('core: a retention backlog is pruned in bounded slices, never thousands of rows in one apply', () => {
120 const core = new CounterCore(sqlite());
121 const t = 1_000_000_000_000;
122 // 2,000 batches and 2,000 hit slots, all past retention by the time the next apply runs (an idle spell).
123 for (let i = 0; i < 2_000; i++) core.apply(req(`old:${i}`, [[i.toString(16).padStart(KEY_HEX, '0'), 0, 1]]), t);
124 assert.deepEqual(core.sizes(), { hits: 2_000, batches: 2_000, blocked: 0 });
125 let now = t + DEDUPE_RETENTION_MS + 60_000;
126 let applies = 0;
127 let maxRows = 0;
128 while (core.sizes().batches > 1 + applies || core.sizes().hits > 1 + applies) {
129 const r = core.apply(req(`new:${applies}`, [[K1, 0, 1]]), now);
130 maxRows = Math.max(maxRows, r.rowsWritten);
131 applies++;
132 now += 10; // far inside PRUNE_EVERY_MS: the backlog, not the timer, keeps the prune going
133 assert.ok(applies < 100, 'the backlog drains');
134 }
135 // One apply: its own few writes + at most PRUNE_BATCH_ROWS per table.
136 assert.ok(maxRows <= 3 * PRUNE_BATCH_ROWS + 5, `max rows written per apply ${maxRows}`);
137 assert.equal(applies, Math.ceil(2_000 / PRUNE_BATCH_ROWS));
138 // Drained: the next apply after the timer prunes nothing old.
139 const quiet = core.apply(req('after', [[K2, 0, 1]]), now + 6_000);
140 assert.ok(quiet.rowsWritten <= 5);
141});
143test('core: rejects what an isolate would never send', () => {
144 const core = new CounterCore(sqlite());
145 const bad: ApplyRequest[] = [
146 req('', [[K1, 0, 1]]),
147 req('x y', [[K1, 0, 1]]),
148 req('ok:1', [['ZZ', 0, 1]]),
149 req('ok:1', [[K1, 2, 1]]),
150 req('ok:1', [[K1, 0, 0]]),
151 { ...req('ok:1', [[K1, 0, 1]]), limits: [0, 20] },
152 ];
153 for (const r of bad) assert.throws(() => core.apply(r, 1), InvalidBatch);
154});
156// ---- a virtual clock for the batcher ----
158class Clock {
159 t = 1_000_000_000_000;
160 timers: { at: number; fire: () => void }[] = [];
161 now = (): number => this.t;
162 sleep = (ms: number): Promise<void> => new Promise(resolve => { this.timers.push({ at: this.t + ms, fire: resolve }); });
163 async advance(ms: number): Promise<void> {
164 const end = this.t + ms;
165 for (;;) {
166 await flushMicrotasks();
167 this.timers.sort((a, b) => a.at - b.at);
168 const next = this.timers[0];
169 if (next === void 0 || next.at > end) break;
170 this.timers.shift();
171 this.t = next.at;
172 next.fire();
173 }
174 this.t = end;
175 await flushMicrotasks();
176 }
179async function flushMicrotasks(): Promise<void> {
180 for (let i = 0; i < 20; i++) await Promise.resolve();
183type Harness = { clock: Clock; core: CounterCore; batcher: Batcher; reports: FlushReport[]; sends: ApplyRequest[] };
185function harness(send: (h: { clock: Clock; core: CounterCore }, r: ApplyRequest) => Promise<ApplyResult>, tuning: Partial<BatcherDeps> = {}): Harness {
186 const clock = new Clock();
187 const core = new CounterCore(sqlite());
188 const reports: FlushReport[] = [];
189 const sends: ApplyRequest[] = [];
190 const batcher = new Batcher({
191 now: clock.now,
192 sleep: clock.sleep,
193 send: r => { sends.push(r); return send({ clock, core }, r); },
194 isolateId: () => 'iso',
195 limits: () => LIMITS,
196 report: r => { reports.push(r); },
197 ...defaultTuning(),
198 ...tuning,
199 });
200 return { clock, core, batcher, reports, sends };
203const instant = ({ clock, core }: { clock: Clock; core: CounterCore }, r: ApplyRequest): Promise<ApplyResult> => Promise.resolve(core.apply(r, clock.now()));
205test('batcher: hits within the linger are one RPC with one id; the next batch has the next id', async () => {
206 const h = harness(instant);
207 h.batcher.add(K1, 0, null);
208 h.batcher.add(K1, 0, null);
209 h.batcher.add(K2, 1, null);
210 assert.equal(h.sends.length, 0, 'nothing is sent before the linger');
211 await h.clock.advance(199);
212 assert.equal(h.sends.length, 0);
213 await h.clock.advance(1);
214 assert.equal(h.sends.length, 1);
215 assert.deepEqual([...h.sends[0].entries].sort(), [[K1, 0, 2], [K2, 1, 1]].sort());
216 assert.equal(h.sends[0].id, 'iso:1');
217 await h.clock.advance(1);
218 assert.equal(h.reports[0].outcome, 'ok');
219 assert.equal(h.reports[0].lingerMs, 200);
220 h.batcher.add(K1, 0, null);
221 await h.clock.advance(250);
222 assert.equal(h.sends[1].id, 'iso:2');
223 assert.equal(h.batcher.inflight, 0);
224});
226test('batcher: a DO that applies after the client gave up, plus the retry, counts once', async () => {
227 // Attempt 1 applies at once but its answer takes 3 s (the client's 1.5 s timer wins); attempt 2 is answered at once.
228 const h = harness(({ clock, core }, r) => {
229 const result = core.apply(r, clock.now());
230 if (r.attempt === 1) return clock.sleep(3_000).then(() => result);
231 return Promise.resolve(result);
232 });
233 for (let i = 0; i < 60; i++) h.batcher.add(K1, 0, null);
234 await h.clock.advance(200 + 1_500 + 50 + 10);
235 const r = h.reports[0];
236 assert.equal(r.outcome, 'ok');
237 assert.deepEqual(r.attempts.map(a => a.outcome), ['timeout', 'ok']);
238 assert.equal(r.attempts[0].ms, 1_500);
239 assert.equal(r.replay, true, 'the retry was acknowledged as a replay');
240 assert.equal(new Set(h.sends.map(s => s.id)).size, 1, 'every attempt carries the same key');
241 // Exactly 60 counted: one more admitted hit makes 61, and the block's peak is 60 before it.
242 const doc = h.core.doc(h.clock.now());
243 assert.equal(doc.entries.length, 1);
244 const more = h.core.apply(req('other:1', [[K1, 0, 1]]), h.clock.now());
245 assert.equal(more.over[0].sum, 61);
246 await h.clock.advance(3_000);
247});
249test('batcher: a slow DO that applies late (after the retry) still counts once', async () => {
250 // Attempt 1 is delivered late: it applies 2 s after it was sent. Attempt 2 applies first. Attempt 1 is then a replay.
251 const h = harness(({ clock, core }, r) => {
252 if (r.attempt === 1) return clock.sleep(2_000).then(() => core.apply(r, clock.now()));
253 return Promise.resolve(core.apply(r, clock.now()));
254 });
255 for (let i = 0; i < 10; i++) h.batcher.add(K1, 0, null);
256 await h.clock.advance(5_000);
257 assert.equal(h.reports[0].outcome, 'ok');
258 assert.equal(h.reports[0].replay, false, 'attempt 2 applied');
259 const after = h.core.apply(req('x:1', [[K1, 0, 50]]), h.clock.now());
260 assert.equal(after.over[0].sum, 60, '10 + 50: the late attempt 1 did not add 10 more');
261});
263test('batcher: errors are retried with backoff and the same id', async () => {
264 let calls = 0;
265 const h = harness(({ clock, core }, r) => {
266 calls++;
267 if (calls <= 2) return Promise.reject(new Error('Network connection lost'));
268 return Promise.resolve(core.apply(r, clock.now()));
269 });
270 h.batcher.add(K1, 0, null);
271 await h.clock.advance(1_000);
272 const r = h.reports[0];
273 assert.deepEqual(r.attempts.map(a => a.outcome), ['error', 'error', 'ok']);
274 assert.equal(r.flushMs, 50 + 100);
275 assert.match(r.attempts[0].error, /Network connection lost/);
276 assert.equal(new Set(h.sends.map(s => s.id)).size, 1);
277});
279test('batcher: 5 s after the first attempt the batch is dropped and recorded (bounded under-count)', async () => {
280 const h = harness(({ clock }) => clock.sleep(60_000).then(() => { throw new Error('never'); }));
281 h.batcher.add(K1, 0, null);
282 h.batcher.add(K1, 0, null);
283 await h.clock.advance(200 + 5_000 + 10);
284 const r = h.reports[0];
285 assert.equal(r.outcome, 'dropped');
286 assert.equal(r.deadlineHit, true);
287 assert.equal(r.hits, 2);
288 assert.ok(r.flushMs <= 5_000, `flush ${r.flushMs}`);
289 assert.ok(r.attempts.length >= 4, `attempts ${r.attempts.length}`);
290 assert.equal(r.attempts.every(a => a.outcome === 'timeout'), true);
291 assert.equal(r.aliveMs, r.flushMs + 200);
292 assert.equal(h.batcher.inflight, 0);
293});
295test('batcher: the accumulator is bounded — a full batch flushes at once; too many in flight are dropped', async () => {
296 const h = harness(({ clock }) => clock.sleep(60_000).then(() => { throw new Error('x'); }), { maxKeys: 3, maxInflight: 2 });
297 h.batcher.add('000000000000000a', 0, null);
298 h.batcher.add('000000000000000b', 0, null);
299 assert.equal(h.sends.length, 0);
300 h.batcher.add('000000000000000c', 0, null);
301 await h.clock.advance(0);
302 assert.equal(h.sends.length, 1, 'the third key closed the batch without waiting for the linger');
303 assert.equal(h.batcher.pending().keys, 0);
304 h.batcher.add('000000000000000d', 0, null);
305 assert.equal(h.batcher.inflight, 2);
306 assert.equal(h.batcher.add('000000000000000e', 0, null) === null, false, 'joins the open batch');
307 h.batcher.add('000000000000000f', 0, null);
308 assert.equal(h.batcher.add('0000000000000010', 0, null), null, 'a third batch is over maxInflight');
309 assert.equal(h.batcher.overflowDrops, 1);
310 await h.clock.advance(6_000);
311});
313test('batcher: an isolate that dies before its flush loses only that batch', async () => {
314 const h = harness(instant);
315 for (let i = 0; i < 7; i++) h.batcher.add(K1, 0, null);
316 await h.clock.advance(250);
317 for (let i = 0; i < 3; i++) h.batcher.add(K1, 0, null);
318 const lost = h.batcher.pending().hits;
319 // The isolate is evicted here: its timers never fire. What reached the DO is everything but the open batch.
320 const counted = h.core.apply(req('probe:1', [[K1, 0, 1]]), h.clock.now());
321 assert.equal(lost, 3);
322 assert.equal(counted.over.length, 0);
323 const sum = h.core.apply(req('probe:2', [[K1, 0, 52]]), h.clock.now()).over[0].sum;
324 assert.equal(sum, 7 + 1 + 52, 'the 3 hits in the open batch are the whole under-count');
325});
327test('batcher: a fault rides only on its own batch, flushed at once', async () => {
328 const h = harness(instant);
329 h.batcher.add(K1, 0, { kind: 'slow-ack', ms: 1500, attempts: 1 });
330 await h.clock.advance(0);
331 assert.equal(h.sends.length, 1, 'no linger: the faulted hit is sent at once');
332 assert.deepEqual(h.sends[0].fault, { kind: 'slow-ack', ms: 1500, attempts: 1 });
333 assert.deepEqual(h.sends[0].entries, [[K1, 0, 1]]);
334 await h.clock.advance(250);
335});
337test('batcher: a faulted hit never joins, and never taints, the shared batch real clients are in', async () => {
338 // Every attempt carrying a fault errors (as the DO does for error;attempts=n): the faulted batch is retried to its
339 // deadline and dropped. The real clients' batch, open in the same isolate at the same time, is unaffected.
340 const h = harness(({ clock, core }, r) => (r.fault !== null ? Promise.reject(new Error('injected')) : Promise.resolve(core.apply(r, clock.now()))));
341 h.batcher.add(K1, 0, null);
342 const faulted = h.batcher.add('000000000000000f', 0, { kind: 'error', ms: 0, attempts: 99 });
343 h.batcher.add(K2, 0, null);
344 assert.equal(h.batcher.pending().hits, 2, 'the open batch holds only the real hits');
345 assert.equal(h.batcher.inflight, 2);
346 await h.clock.advance(6_000);
347 const shared = h.reports.find(r => r.fault === null);
348 const own = h.reports.find(r => r.fault !== null);
349 assert.ok(shared && own);
350 assert.equal(shared.outcome, 'ok');
351 assert.equal(shared.hits, 2);
352 assert.deepEqual(shared.attempts.map(a => a.outcome), ['ok']);
353 assert.equal(own.outcome, 'dropped');
354 assert.equal(own.hits, 1);
355 assert.equal(own.keys, 1);
356 const faultedReport = await faulted;
357 assert.ok(faultedReport);
358 assert.equal(faultedReport.id, own.id);
359 assert.notEqual(own.id, shared.id);
360 assert.equal(h.sends.filter(s => s.fault === null).every(s => s.entries.every(e => e[0] !== '000000000000000f')), true);
361 // The real counts reached the counter exactly once.
362 assert.equal(h.core.apply(req('probe:x', [[K1, 0, 58]]), h.clock.now()).over.length, 0);
363 assert.equal(h.core.apply(req('probe:y', [[K1, 0, 1]]), h.clock.now()).over[0].sum, 60);
364});
366test('batcher: an isolated (faulted) batch respects the in-flight bound', async () => {
367 const h = harness(({ clock }) => clock.sleep(60_000).then(() => { throw new Error('x'); }), { maxInflight: 1 });
368 assert.notEqual(h.batcher.add(K1, 0, { kind: 'error', ms: 0, attempts: 9 }), null);
369 assert.equal(h.batcher.add(K2, 0, { kind: 'error', ms: 0, attempts: 9 }), null);
370 assert.equal(h.batcher.overflowDrops, 1);
371 await h.clock.advance(6_000);
372});
374test('fault header parse', () => {
375 assert.deepEqual(parseFault('slow-ack=3000;attempts=2'), { kind: 'slow-ack', ms: 3000, attempts: 2 });
376 assert.deepEqual(parseFault('slow-ack=3000'), { kind: 'slow-ack', ms: 3000, attempts: 1 });
377 assert.deepEqual(parseFault('error;attempts=3'), { kind: 'error', ms: 0, attempts: 3 });
378 assert.equal(parseFault('drop table'), null);
379 assert.equal(parseFault(null), null);
380});
382// ---- the document ----
384test('doc: served with max-age + SWR, one path only, and indexed by key and bucket', async () => {
385 const core = new CounterCore(sqlite());
386 core.apply(req('d:1', [[K1, 0, 60]]), 1_000_000_000_000);
387 const source = { blockedDoc: async () => core.doc(1_000_000_000_000) };
388 const res = await serveDoc(new Request(DOC_URL), source);
389 assert.equal(res.status, 200);
390 assert.equal(res.headers.get('cache-control'), 'public, max-age=1, stale-while-revalidate=30');
391 assert.match(String(res.headers.get('etag')), /^"g\d+"$/);
392 const d = indexDoc(await res.text());
393 assert.equal(d.index.has(`${K1}:0`), true);
394 assert.equal((await serveDoc(new Request(`https://rate.internal${DOC_PATH}x`), source)).status, 404);
395 assert.equal((await serveDoc(new Request(DOC_URL, { method: 'POST' }), source)).status, 405);
396 const failing = await serveDoc(new Request(DOC_URL), { blockedDoc: () => Promise.reject(new Error('down')) });
397 assert.equal(failing.status, 503);
398 assert.equal(failing.headers.get('cache-control'), 'no-store');
399});
401// ---- identity ----
403test('identity: load-test headers count only with the right token', async () => {
404 const env = { PLY_LOAD_TOKEN: 'secret-token-123', RATE_MODE: 'legacy', RATE_COUNTER: fakeNamespace(new CounterCore(sqlite())) };
405 const plain = await rateIdentity(new Request('https://x/', { headers: { 'cf-connecting-ip': '198.51.100.1', [LOAD_CLIENT_HEADER]: 'c1', [LOAD_MODE_HEADER]: 'batched' } }), env, null);
406 assert.equal(plain.load, false);
407 assert.equal(plain.mode, 'legacy');
408 assert.equal(plain.modeSource, 'default');
409 const wrong = await rateIdentity(new Request('https://x/', { headers: { [LOAD_TOKEN_HEADER]: 'nope', [LOAD_CLIENT_HEADER]: 'c1', [LOAD_MODE_HEADER]: 'batched' } }), env, null);
410 assert.equal(wrong.load, false);
411 const right = await rateIdentity(new Request('https://x/', { headers: { [LOAD_TOKEN_HEADER]: 'secret-token-123', [LOAD_CLIENT_HEADER]: 'c1', [LOAD_MODE_HEADER]: 'batched' } }), env, null);
412 assert.equal(right.load, true);
413 assert.equal(right.mode, 'batched');
414 assert.equal(right.modeSource, 'header');
415 const other = await rateIdentity(new Request('https://x/', { headers: { [LOAD_TOKEN_HEADER]: 'secret-token-123', [LOAD_CLIENT_HEADER]: 'c2' } }), env, null);
416 assert.notEqual(right.hex, other.hex);
417 assert.equal(other.mode, 'legacy');
418 assert.equal(other.modeSource, 'default', 'a load request with no mode header runs the default, like a real one');
419 assert.equal(right.hex.length, 64);
420});
422test('identity: PLY_TOKEN (the /v1 bearer) does not unlock the load-test headers', async () => {
423 const fault = { 'x-ply-rate-fault': 'error;attempts=3' };
424 const apiOnly = { PLY_TOKEN: 'api-bearer-456', RATE_MODE: 'batched', RATE_COUNTER: fakeNamespace(new CounterCore(sqlite())) };
425 const withApi = await rateIdentity(new Request('https://x/', { headers: { 'cf-connecting-ip': '198.51.100.7', [LOAD_TOKEN_HEADER]: 'api-bearer-456', [LOAD_CLIENT_HEADER]: 'c1', [LOAD_MODE_HEADER]: 'legacy', ...fault } }), apiOnly, null);
426 assert.equal(withApi.load, false, 'no PLY_LOAD_TOKEN bound: nothing is a load request, whatever is presented');
427 assert.equal(withApi.mode, 'batched');
428 assert.equal(withApi.fault, null);
429 const plainIp = await rateIdentity(new Request('https://x/', { headers: { 'cf-connecting-ip': '198.51.100.7' } }), apiOnly, null);
430 assert.equal(withApi.hex, plainIp.hex, 'keyed by the address, the label ignored');
431 const both = { ...apiOnly, PLY_LOAD_TOKEN: 'load-only-789' };
432 const apiPresented = await rateIdentity(new Request('https://x/', { headers: { [LOAD_TOKEN_HEADER]: 'api-bearer-456', [LOAD_MODE_HEADER]: 'legacy', ...fault } }), both, null);
433 assert.equal(apiPresented.load, false, 'PLY_TOKEN presented while PLY_LOAD_TOKEN is bound: still refused');
434 assert.equal(apiPresented.fault, null);
435 const loadPresented = await rateIdentity(new Request('https://x/', { headers: { [LOAD_TOKEN_HEADER]: 'load-only-789', [LOAD_MODE_HEADER]: 'legacy', ...fault } }), both, null);
436 assert.equal(loadPresented.load, true);
437 assert.equal(loadPresented.mode, 'legacy');
438 assert.deepEqual(loadPresented.fault, { kind: 'error', ms: 0, attempts: 3 });
439 const empty = await rateIdentity(new Request('https://x/', { headers: { [LOAD_TOKEN_HEADER]: '' } }), { ...apiOnly, PLY_LOAD_TOKEN: '' }, null);
440 assert.equal(empty.load, false, 'an empty PLY_LOAD_TOKEN matches nothing');
441});
443// ---- end to end with a fake ctx: Workers Cache emulated with max-age, the DO as CounterCore ----
445function fakeNamespace(core: CounterCore, clock: { now(): number } = { now: () => Date.now() }) {
446 const stub = {
447 apply: async (r: ApplyRequest) => core.apply(r, clock.now()),
448 blockedDoc: async () => core.doc(clock.now()),
449 };
450 return { idFromName: (n: string) => n, get: () => stub };
453test('admit: the client over its limit is refused once its batch is flushed and the cached document refreshes', async () => {
454 resetRateState();
455 const core = new CounterCore(sqlite());
456 const ns = fakeNamespace(core);
457 const env = { RATE_MODE: 'batched', RATE_COUNTER: ns, PLY_LOAD_TOKEN: 't' };
458 let cached: { body: string; etag: string; at: string; storedAt: number } | null = null;
459 let docReads = 0;
460 const waits: Promise<unknown>[] = [];
461 const ctx = {
462 waitUntil: (p: Promise<unknown>) => { waits.push(p); },
463 exports: {
464 RateCache: {
465 fetch: async (r: Request) => {
466 if (cached !== null && Date.now() - cached.storedAt < 1_000) return new Response(cached.body, { headers: { etag: cached.etag, 'x-ply-rate-at': cached.at, 'cf-cache-status': 'HIT' } });
467 docReads++;
468 const res = await serveDoc(r, ns.get());
469 const body = await res.text();
470 cached = { body, etag: String(res.headers.get('etag')), at: String(res.headers.get('x-ply-rate-at')), storedAt: Date.now() };
471 return new Response(body, { headers: { etag: cached.etag, 'x-ply-rate-at': cached.at, 'cf-cache-status': 'MISS' } });
472 },
473 },
474 },
475 };
476 const id = { hex: 'c'.repeat(64), mode: 'batched' as const, fault: null, load: true, modeSource: 'default' as const };
477 const limitFor = (b: 'page' | 'pack') => (b === 'page' ? 3 : 20);
478 const attrs: Record<string, string | number | boolean> = {};
479 const set = (k: string, v: string | number | boolean) => { attrs[k] = v; };
480 for (let i = 0; i < 5; i++) {
481 const d = await admitBatched({ ctx, env, id, bucket: 'page', now: Date.now(), limitFor, set });
482 assert.equal(d.ok, true, 'within the lag every request is admitted (the overshoot)');
483 }
484 assert.equal(waits.length, 1, 'only the request that opened the batch waits for it');
485 await Promise.all(waits);
486 assert.equal(attrs['rate.cache'], 'HIT');
487 await new Promise(r => setTimeout(r, 1_050));
488 const denied = await admitBatched({ ctx, env, id, bucket: 'page', now: Date.now(), limitFor, set });
489 assert.equal(denied.ok, false);
490 assert.ok(denied.retryAfterMs > 50_000);
491 assert.equal(attrs['rate.decision'], 'deny');
492 assert.equal(attrs['rate.cache'], 'MISS');
493 assert.equal(docReads, 2);
494 const other = { ...id, hex: 'd'.repeat(64) };
495 assert.equal((await admitBatched({ ctx, env, id: other, bucket: 'page', now: Date.now(), limitFor, set })).ok, true);
496 assert.equal(core.doc(Date.now()).entries[0][0], 'c'.repeat(KEY_HEX));
497 await Promise.all(waits);
498});
500test('admit: a failed read uses the last good document, and with none admits (fail open)', async () => {
501 resetRateState();
502 const env = { RATE_MODE: 'batched', RATE_COUNTER: fakeNamespace(new CounterCore(sqlite())) };
503 const ctx = { waitUntil: () => {}, exports: { RateCache: { fetch: async () => { throw new Error('down'); } } } };
504 const r = await readDoc(ctx, Date.now());
505 assert.equal(r.read, 'fail');
506 const attrs: Record<string, string | number | boolean> = {};
507 const d = await admitBatched({ ctx, env, id: { hex: 'e'.repeat(64), mode: 'batched', fault: null, load: false, modeSource: 'default' }, bucket: 'page', now: Date.now(), limitFor: () => 60, set: (k, v) => { attrs[k] = v; } });
508 assert.equal(d.ok, true);
509 assert.equal(attrs['rate.decision'], 'fail_open');
510 resetRateState();
511});
513// ---- the deployed bundle under workerd: real RateCounter on SQLite, ctx.exports.RateCache, the load-test headers ----
515function freePort(): Promise<number> {
516 return new Promise((resolve, reject) => {
517 const srv = createServer();
518 srv.once('error', reject);
519 srv.listen(0, '127.0.0.1', () => {
520 const addr = srv.address();
521 const port = addr !== null && typeof addr === 'object' ? addr.port : 0;
522 srv.close(() => resolve(port));
523 });
524 });
527const WORKERD = [process.env.WORKERD, fileURLToPath(new URL('../node_modules/.bin/workerd', import.meta.url))].filter((p): p is string => !!p && existsSync(p));
529async function workerdRun(withBinding: boolean): Promise<void> {
530 const root = mkdtempSync(join(tmpdir(), 'ply-rate-'));
531 const out = join(root, 'wk');
532 mkdirSync(out);
533 copyCfBundle(join(out, 'worker.mjs'));
534 const port = await freePort();
535 const token = 'load-token-' + 'y'.repeat(20);
536 writeFileSync(join(out, 'config.capnp'), `using Workerd = import "/workerd/workerd.capnp";
537const config :Workerd.Config = (
538 services = [ (name = "main", worker = (
539 modules = [ (name = "worker.mjs", esModule = embed "worker.mjs") ],
540 compatibilityDate = "2026-10-18",
541 compatibilityFlags = ["nodejs_als"],
542 durableObjectNamespaces = [ (className = "ViewerRateLimit", uniqueKey = "viewer-rate-test", enableSql = true), (className = "RateCounter", uniqueKey = "rate-counter-test", enableSql = true) ],
543 durableObjectStorage = (localDisk = "do-disk"),
544 bindings = [
545 (name = "PLY_TOKEN", text = "api-${token}"),
546 (name = "PLY_LOAD_TOKEN", text = "${token}"),
547 (name = "WRITES", text = "off"),
548 (name = "PACK_CACHE", text = "off"),
549 (name = "VIEWER_RATE_PER_MIN", text = "3"),
550 (name = "RATE_MODE", text = "batched"),
551 (name = "VIEWER_RATE", durableObjectNamespace = "ViewerRateLimit")${withBinding ? ', (name = "RATE_COUNTER", durableObjectNamespace = "RateCounter")' : ''}
552 ]
553 )), (name = "do-disk", disk = (path = "${join(root, 'do')}", writable = true)), (name = "internet", network = (allow = ["private", "public", "local"])) ],
554 sockets = [ (name = "http", address = "127.0.0.1:${port}", http = (), service = "main") ]
555);
556`);
557 mkdirSync(join(root, 'do'));
558 let logs = '';
559 const proc = spawn(WORKERD[0] as string, ['serve', join(out, 'config.capnp'), '--verbose'], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
560 proc.stdout.on('data', d => { logs += d; });
561 proc.stderr.on('data', d => { logs += d; });
562 const sleep = (ms: number) => new Promise(r => setTimeout(r, ms));
563 try {
564 let up = false;
565 for (let i = 0; i < 80 && !up; i++) {
566 try { up = (await fetch(`http://127.0.0.1:${port}/`)).ok; } catch { await sleep(100); }
567 }
568 assert.equal(up, true, logs.slice(-1500));
569 const hit = (client: string, extra: Record<string, string> = {}) => fetch(`http://127.0.0.1:${port}/view`, { headers: { [LOAD_TOKEN_HEADER]: token, [LOAD_CLIENT_HEADER]: client, ...extra } });
570 // Limit 3. The batch flushes 200 ms after the first request; workerd has no Workers Cache, so the next read sees it.
571 for (let i = 0; i < 3; i++) assert.equal((await hit('c1')).status, 200);
572 await sleep(600);
573 const refused = await hit('c1');
574 assert.equal(refused.status, 429, `${await refused.text()}\n${logs.slice(-3000)}`);
575 assert.ok(Number(refused.headers.get('retry-after')) >= 50);
576 assert.equal((await hit('c2')).status, 200, 'another client is not refused');
577 // Legacy arm through the same headers: its own per-client object.
578 for (let i = 0; i < 3; i++) assert.equal((await hit('c3', { [LOAD_MODE_HEADER]: 'legacy' })).status, 200);
579 assert.equal((await hit('c3', { [LOAD_MODE_HEADER]: 'legacy' })).status, 429);
580 // Memo arm: decides from the isolate's document with no fetch. The first request after the flush still decides
581 // from a document read before it (the memo's lag) and refreshes it; a later one is refused.
582 for (let i = 0; i < 3; i++) assert.equal((await hit('c5', { [LOAD_MODE_HEADER]: 'memo' })).status, 200);
583 await sleep(700);
584 assert.equal((await hit('c5', { [LOAD_MODE_HEADER]: 'memo' })).status, 200, 'decided from the stale memo; refresh started');
585 await sleep(300);
586 const memoRefused = await hit('c5', { [LOAD_MODE_HEADER]: 'memo' });
587 assert.equal(memoRefused.status, 429);
588 assert.ok(Number(memoRefused.headers.get('retry-after')) >= 50);
589 // A forged token is ignored: the headers do nothing and the request is keyed by its (absent) address.
590 const forged = await fetch(`http://127.0.0.1:${port}/view`, { headers: { [LOAD_TOKEN_HEADER]: 'nope', [LOAD_CLIENT_HEADER]: 'c1' } });
591 assert.equal(forged.status, 200);
592 // The /v1 bearer is not the load token: presented as one, the label is ignored (keyed by the absent address, which
593 // the forged request above already counted once), so c1's block does not apply to it.
594 const apiAsLoad = await fetch(`http://127.0.0.1:${port}/view`, { headers: { [LOAD_TOKEN_HEADER]: `api-${token}`, [LOAD_CLIENT_HEADER]: 'c1' } });
595 assert.equal(apiAsLoad.status, 200, 'PLY_TOKEN does not unlock x-ply-rate-client');
596 if (!withBinding) return;
597 // workerd prints a Durable Object's console only when it is reached through an env binding, so the fault half runs there.
598 // Fault: the DO applies and holds its answer 1.5 s; the 1 s attempt timer fires, the retry with the same id is a replay.
599 assert.equal((await hit('c4', { 'x-ply-rate-fault': 'slow-ack=1500;attempts=1' })).status, 200);
600 await sleep(2_500);
601 const applies = logs.split('\n').filter(l => l.includes('"ply.rate.apply"') && l.includes('slow-ack') === false).map(l => JSON.parse(l.slice(l.indexOf('{'))));
602 const faulted = logs.split('\n').filter(l => l.includes('"ply.rate.apply"') && l.includes('slow-ack=1500')).map(l => JSON.parse(l.slice(l.indexOf('{'))));
603 assert.equal(faulted.length, 1, logs.split('\n').filter(l => !l.includes('ply.request')).join('\n').slice(-3000));
604 const replays = applies.filter(a => a.id === faulted[0].id);
605 assert.equal(replays.length, 1, `one retry of the faulted batch\n${logs.split('\n').filter(l => l.includes('rate') || l.includes('rror')).join('\n').slice(-4000)}`);
606 assert.equal(replays[0].replay, true);
607 assert.equal(replays[0].attempt, 2);
608 assert.equal(logs.includes(token), false, 'the load token never reaches a log');
609 assert.equal(faulted[0].keys, 1, 'the faulted request flushed in a batch of its own');
610 } finally {
611 if (process.env.RATE_TEST_LOG) writeFileSync(process.env.RATE_TEST_LOG, logs);
612 if (proc.pid) { try { process.kill(-proc.pid, 'SIGKILL'); } catch { proc.kill('SIGKILL'); } }
613 else proc.kill('SIGKILL');
614 rmSync(root, { recursive: true, force: true });
615 }
618test('workerd: batched limiter through ctx.exports.RateCounter (production path) refuses within the lag', { skip: WORKERD.length === 0 && 'workerd not installed', timeout: 60_000 }, () => workerdRun(false));
620test('workerd: a slow ack is retried with the same id and applied once (replay)', { skip: WORKERD.length === 0 && 'workerd not installed', timeout: 60_000 }, () => workerdRun(true));
622test('counterNamespace: found on a ctx.exports that answers `in` with false (the platform object does)', () => {
623 const ns = fakeNamespace(new CounterCore(sqlite()));
624 const exportsLike = new Proxy({ RateCounter: ns }, { has: () => false });
625 assert.equal('RateCounter' in exportsLike, false);
626 assert.equal(counterNamespace({}, { exports: exportsLike }), ns);
627 assert.equal(counterNamespace({}, { exports: {} }), null);
628 // The platform's namespace is callable: typeof 'function', with idFromName.
629 const callable = Object.assign(() => {}, ns);
630 assert.equal(counterNamespace({}, { exports: { RateCounter: callable } }), callable);
631 assert.equal(counterNamespace({}, null), null);
632});
634test('admit: the read never waits past its budget — late reads use the isolate memo, or admit with none', async () => {
635 resetRateState();
636 const core = new CounterCore(sqlite());
637 core.apply(req('b:1', [['f'.repeat(KEY_HEX), 0, 60]]), Date.now());
638 const ns = fakeNamespace(core);
639 const env = { RATE_MODE: 'batched', RATE_COUNTER: ns };
640 const waits: Promise<unknown>[] = [];
641 // Every read is a slow MISS: 100 ms, four times the budget.
642 const ctx = { waitUntil: (p: Promise<unknown>) => { waits.push(p); }, exports: { RateCache: { fetch: async (r: Request) => { await new Promise(res => setTimeout(res, 100)); return serveDoc(r, ns.get()); } } } };
643 const id = { hex: 'f'.repeat(64), mode: 'batched' as const, fault: null, load: true, modeSource: 'default' as const };
644 const attrs: Record<string, string | number | boolean> = {};
645 const set = (k: string, v: string | number | boolean) => { attrs[k] = v; };
646 const t0 = Date.now();
647 const first = await admitBatched({ ctx, env, id, bucket: 'page', now: Date.now(), limitFor: () => 60, set });
648 assert.ok(Date.now() - t0 < 80, 'did not wait for the slow read');
649 assert.equal(first.ok, true, 'no document in this isolate yet: admitted (the bounded extra lag)');
650 assert.equal(attrs['rate.read'], 'late_open');
651 await Promise.all(waits);
652 const second = await admitBatched({ ctx, env, id, bucket: 'page', now: Date.now(), limitFor: () => 60, set });
653 assert.equal(second.ok, false, 'the late fetch refreshed the isolate document; this read decides from it');
654 assert.equal(attrs['rate.read'], 'late_memo');
655 await Promise.all(waits);
656 resetRateState();
657});
659// ---- memo mode: the decision is a lookup into the isolate's document; refreshes are single-flight in waitUntil ----
661function memoRig(fetchDelayMs: number) {
662 const core = new CounterCore(sqlite());
663 const ns = fakeNamespace(core);
664 const env = { RATE_MODE: 'memo', RATE_COUNTER: ns };
665 const waits: Promise<unknown>[] = [];
666 let fetches = 0;
667 const ctx = {
668 waitUntil: (p: Promise<unknown>) => { waits.push(p); },
669 exports: { RateCache: { fetch: async (r: Request) => { fetches++; await new Promise(res => setTimeout(res, fetchDelayMs)); return serveDoc(r, ns.get()); } } },
670 };
671 const attrs: Record<string, string | number | boolean> = {};
672 const set = (k: string, v: string | number | boolean) => { attrs[k] = v; };
673 const id = (c: string) => ({ hex: c.repeat(64), mode: 'memo' as const, fault: null, load: true, modeSource: 'header' as const });
674 const admit = (c: string) => admitBatched({ ctx, env, id: id(c), bucket: 'page', now: Date.now(), limitFor: () => 60, set });
675 return { core, env, ctx, waits, attrs, admit, fetches: () => fetches, settle: () => Promise.all(waits) };
678test('memo: a cold isolate admits at once (fail_open, read cold) and refreshes in waitUntil; then denies from the memo', async () => {
679 resetRateState();
680 const rig = memoRig(100);
681 rig.core.apply(req('m:1', [['a'.repeat(KEY_HEX), 0, 60]]), Date.now());
682 const t0 = Date.now();
683 const first = await rig.admit('a');
684 assert.ok(Date.now() - t0 < 50, 'no fetch on the request path');
685 assert.equal(first.ok, true, 'cold: admitted, though the document would refuse it');
686 assert.equal(rig.attrs['rate.read'], 'cold');
687 assert.equal(rig.attrs['rate.decision'], 'fail_open');
688 assert.equal(rig.attrs['rate.refresh.started'], true);
689 await rig.settle();
690 const second = await rig.admit('a');
691 assert.equal(second.ok, false, 'deny from the memo');
692 assert.equal(rig.attrs['rate.read'], 'memo_fresh');
693 assert.equal(rig.attrs['rate.decision'], 'deny');
694 assert.ok(second.retryAfterMs > 50_000);
695 assert.equal((await rig.admit('b')).ok, true, 'another client is admitted from the same memo');
696 assert.equal(rig.fetches(), 1, 'fresh memo: no further reads');
697 await rig.settle();
698 resetRateState();
699});
701test('memo: cold while the first read is in flight is isolate_start (rate.cold); cold after a failed read is read_failed', async () => {
702 resetRateState();
703 const rig = memoRig(60);
704 await rig.admit('a');
705 assert.equal(rig.attrs['rate.cold'], 'isolate_start', 'the isolate\'s first check');
706 assert.equal(rig.attrs['rate.fail_open_reason'], 'isolate_start');
707 delete rig.attrs['rate.cold'];
708 await rig.admit('b');
709 assert.equal(rig.attrs['rate.read'], 'cold');
710 assert.equal(rig.attrs['rate.cold'], 'isolate_start', 'joined the first read, still start-up');
711 await rig.settle();
712 resetRateState();
713 // A broken read path: the first refresh settles with no document. Later cold checks are degradation, not start-up.
714 const env = { RATE_MODE: 'memo', RATE_COUNTER: fakeNamespace(new CounterCore(sqlite())) };
715 const waits: Promise<unknown>[] = [];
716 const ctx = { waitUntil: (p: Promise<unknown>) => { waits.push(p); }, exports: { RateCache: { fetch: async () => new Response('no', { status: 503 }) } } };
717 const attrs: Record<string, string | number | boolean> = {};
718 const set = (k: string, v: string | number | boolean) => { attrs[k] = v; };
719 const id = { hex: 'c'.repeat(64), mode: 'memo' as const, fault: null, load: false, modeSource: 'default' as const };
720 await admitBatched({ ctx, env, id, bucket: 'page', now: Date.now(), limitFor: () => 60, set });
721 assert.equal(attrs['rate.cold'], 'isolate_start');
722 await Promise.all(waits);
723 delete attrs['rate.cold'];
724 await admitBatched({ ctx, env, id, bucket: 'page', now: Date.now(), limitFor: () => 60, set });
725 assert.equal(attrs['rate.read'], 'cold');
726 assert.equal(attrs['rate.decision'], 'fail_open');
727 assert.equal(attrs['rate.fail_open_reason'], 'read_failed');
728 assert.equal(attrs['rate.cold_last_read'], 'fail');
729 assert.equal('rate.cold' in attrs, false, 'the first read settled with nothing: a warm fail_open the CD gate counts');
730 await Promise.all(waits);
731 resetRateState();
732});
734// ply-web#156: CD 2026-11-12T13-06-51-511Z, isolate 2058502eefa348bb9abc idle 339.4 s after the canary probes, then a
735// crawler request (cf-ray a595ccf10c639249) read cold and failed open; its refresh read ok in 6 ms.
736test('memo: an idle isolate whose memo aged past MEMO_FALLBACK_MS is memo_expired (expected), and its refresh carries the cause', async () => {
737 resetRateState();
738 const rig = memoRig(1);
739 const spans: { name: string; attrs: Record<string, string | number | boolean> }[] = [];
740 const tracing = { startActiveSpan: async <T>(name: string, fn: (span: { setAttribute(k: string, v: string | number | boolean): void; end(): void; setStatus(): void; recordException(): void }) => Promise<T>) => {
741 const attrs: Record<string, string | number | boolean> = {};
742 spans.push({ name, attrs });
743 return fn({ setAttribute: (k, v) => { attrs[k] = v; }, end: () => {}, setStatus: () => {}, recordException: () => {} });
744 } };
745 const ctx = { ...rig.ctx, tracing };
746 await admitBatched({ ctx, env: rig.env, id: { hex: 'a'.repeat(64), mode: 'memo', fault: null, load: false, modeSource: 'default' }, bucket: 'page', now: Date.now(), limitFor: () => 60, set: () => {} });
747 await rig.settle();
748 const realNow = Date.now;
749 const idle = 339_400;
750 try {
751 const base = realNow();
752 Date.now = () => base + idle;
753 const attrs: Record<string, string | number | boolean> = {};
754 const set = (k: string, v: string | number | boolean) => { attrs[k] = v; };
755 const d = await admitBatched({ ctx, env: rig.env, id: { hex: 'b'.repeat(64), mode: 'memo', fault: null, load: false, modeSource: 'default' }, bucket: 'page', now: base + idle, limitFor: () => 60, set });
756 assert.equal(d.ok, true);
757 assert.equal(attrs['rate.read'], 'cold');
758 assert.equal(attrs['rate.decision'], 'fail_open');
759 assert.equal(attrs['rate.cold'], 'memo_expired');
760 assert.equal(attrs['rate.fail_open_reason'], 'memo_expired');
761 assert.ok(Number(attrs['rate.memo_expired_ms']) >= idle - 50, 'the expired memo\'s age is stamped');
762 assert.equal(attrs['rate.refresh.started'], true);
763 } finally {
764 Date.now = realNow;
765 }
766 await rig.settle();
767 const refresh = spans.filter(s => s.name === 'ply.rate.refresh').pop();
768 assert.ok(refresh !== void 0 && refresh.attrs['rate.refresh.trigger'] === 'cold');
769 assert.equal(refresh.attrs['rate.refresh.cold'], 'memo_expired');
770 assert.equal(refresh.attrs['rate.refresh.read'], 'ok', 'healthy: the CD gate keeps it out of the warm share');
771 resetRateState();
772});
774test('memo: an expired memo after a failed refresh is read_failed, not memo_expired', () => {
775 const memo = { etag: '', index: new Map(), gen: 1, at: 0, entries: 0, fetchedAt: 0, bytes: 0, truncated: false };
776 assert.deepEqual(coldCause(memo, { kind: 'ok' }, MEMO_FALLBACK_MS + 1), { kind: 'memo_expired', memoAgeMs: MEMO_FALLBACK_MS + 1 });
777 assert.deepEqual(coldCause(memo, { kind: 'failed', read: 'memo' }, MEMO_FALLBACK_MS + 1), { kind: 'read_failed', lastRead: 'memo', memoAgeMs: MEMO_FALLBACK_MS + 1 });
778 assert.deepEqual(coldCause(memo, { kind: 'failed', read: 'abandoned' }, MEMO_FALLBACK_MS + 1), { kind: 'read_failed', lastRead: 'abandoned', memoAgeMs: MEMO_FALLBACK_MS + 1 });
779 assert.deepEqual(coldCause(null, { kind: 'failed', read: 'fail' }, 5), { kind: 'read_failed', lastRead: 'fail', memoAgeMs: null });
780 assert.deepEqual(coldCause(null, { kind: 'none' }, 5), { kind: 'isolate_start' });
781});
783test('memo: refresh is single-flight — concurrent stale requests join the one in flight', async () => {
784 resetRateState();
785 const rig = memoRig(80);
786 const a = refreshMemo(rig.ctx, rig.env, COLD_START);
787 assert.equal(a.started, true);
788 const joined = [refreshMemo(rig.ctx, rig.env, STALE), refreshMemo(rig.ctx, rig.env, STALE), refreshMemo(rig.ctx, rig.env, STALE)];
789 for (const j of joined) assert.equal(j.started, false);
790 const report = await a.promise;
791 assert.equal(rig.fetches(), 1);
792 assert.equal(report.joins, 3);
793 assert.equal(report.read, 'ok');
794 for (const j of joined) assert.equal(await j.promise, report, 'joiners see the same refresh');
795 assert.equal(rig.waits.length, 4, 'every joiner keeps the flight alive with waitUntil');
796 // Settled: the next stale request starts a new flight.
797 assert.equal(refreshMemo(rig.ctx, rig.env, STALE).started, true);
798 await rig.settle();
799 assert.equal(rig.fetches(), 2);
800 // Through admit: stale memo, many requests, one fetch.
801 const realNow = Date.now;
802 try {
803 const base = realNow();
804 Date.now = () => base + MEMO_REFRESH_MS + 1;
805 const before = rig.fetches();
806 await Promise.all([rig.admit('c'), rig.admit('d'), rig.admit('e'), rig.admit('f')]);
807 assert.equal(rig.attrs['rate.read'], 'memo_stale_refresh');
808 Date.now = realNow;
809 await rig.settle();
810 assert.equal(rig.fetches() - before, 1);
811 } finally {
812 Date.now = realNow;
813 }
814 resetRateState();
815});
817test('memo: age bounds — fresh below MEMO_REFRESH_MS, stale-and-refresh above, no memo beyond MEMO_FALLBACK_MS', async () => {
818 resetRateState();
819 const rig = memoRig(1);
820 rig.core.apply(req('m:2', [['a'.repeat(KEY_HEX), 0, 60]]), Date.now());
821 const fetchedBy = Date.now();
822 await refreshMemo(rig.ctx, rig.env, COLD_START).promise;
823 const fresh = readMemo(rig.ctx, rig.env, fetchedBy + MEMO_REFRESH_MS - 50);
824 assert.equal(fresh.read, 'memo_fresh');
825 assert.equal(rig.fetches(), 1);
826 const stale = readMemo(rig.ctx, rig.env, Date.now() + MEMO_REFRESH_MS);
827 assert.equal(stale.read, 'memo_stale_refresh');
828 if (stale.read !== 'memo_stale_refresh') return;
829 assert.equal(stale.started, true);
830 assert.equal(stale.memo.entries, 1, 'a stale memo still decides');
831 await rig.settle();
832 const expired = readMemo(rig.ctx, rig.env, Date.now() + MEMO_FALLBACK_MS + 1);
833 assert.equal(expired.read, 'cold', 'older than the 5 min bound: treated as no memo');
834 if (expired.read !== 'cold') return;
835 assert.equal(expired.cause.kind, 'memo_expired', 'the last refresh read ok: the memo expired while idle');
836 await rig.settle();
837 resetRateState();
838});
840test('memo: a hung refresh is abandoned after MEMO_REFRESH_ABANDON_MS, so the flight cannot pin forever', async () => {
841 resetRateState();
842 let calls = 0;
843 const ctx = { waitUntil: () => {}, exports: { RateCache: { fetch: () => { calls++; return new Promise<Response>(() => {}); } } } };
844 const env = { RATE_MODE: 'memo', RATE_COUNTER: fakeNamespace(new CounterCore(sqlite())) };
845 assert.equal(refreshMemo(ctx, env, COLD_START).started, true);
846 assert.equal(refreshMemo(ctx, env, COLD_START).started, false);
847 const realNow = Date.now;
848 const base = realNow();
849 try {
850 Date.now = () => base + MEMO_REFRESH_ABANDON_MS + 1;
851 assert.equal(refreshMemo(ctx, env, COLD_START).started, true);
852 } finally {
853 Date.now = realNow;
854 }
855 await new Promise(r => setTimeout(r, 5));
856 assert.equal(calls, 2);
857 resetRateState();
858});
860test('identity: x-ply-rate-mode memo with the token; RATE_MODE=memo as the default', async () => {
861 const ns = fakeNamespace(new CounterCore(sqlite()));
862 const env = { PLY_LOAD_TOKEN: 'secret-token-123', RATE_MODE: 'batched', RATE_COUNTER: ns };
863 const asked = await rateIdentity(new Request('https://x/', { headers: { [LOAD_TOKEN_HEADER]: 'secret-token-123', [LOAD_MODE_HEADER]: 'memo' } }), env, null);
864 assert.equal(asked.mode, 'memo');
865 const forged = await rateIdentity(new Request('https://x/', { headers: { [LOAD_TOKEN_HEADER]: 'nope', [LOAD_MODE_HEADER]: 'memo' } }), env, null);
866 assert.equal(forged.mode, 'batched');
867 const byEnv = await rateIdentity(new Request('https://x/'), { ...env, RATE_MODE: 'memo' }, null);
868 assert.equal(byEnv.mode, 'memo');
869 const unbound = await rateIdentity(new Request('https://x/'), { RATE_MODE: 'memo' }, null);
870 assert.equal(unbound.mode, 'legacy');
871});
873// ---- memory and storage bounds under many distinct clients (a million client addresses a minute) ----
875test('batcher: distinct keys held across open + flushing batches are bounded; a new key past it is dropped (budget)', async () => {
876 let release: () => void = () => {};
877 const gate = new Promise<void>(r => { release = r; });
878 const h = harness(async ({ clock, core }, r) => { await gate; return core.apply(r, clock.now()); }, { maxKeys: 2, maxKeysHeld: 3 });
879 const key = (i: number) => i.toString(16).padStart(KEY_HEX, '0');
880 assert.ok(h.batcher.add(key(1), 0, null));
881 assert.ok(h.batcher.add(key(2), 0, null), 'fills batch 1 (maxKeys 2): it flushes and stays held until acked');
882 assert.ok(h.batcher.add(key(3), 0, null));
883 assert.equal(h.batcher.keysHeld, 3);
884 assert.equal(h.batcher.add(key(4), 0, null), null, 'a fourth distinct key is over the budget');
885 assert.equal(h.batcher.lastDrop, 'budget');
886 assert.equal(h.batcher.budgetDrops, 1);
887 assert.ok(h.batcher.add(key(3), 0, null), 'a key already in the open batch still counts (no new memory)');
888 assert.equal(h.batcher.keysHeld, 3);
889 release();
890 await h.clock.advance(1_000);
891 await new Promise(r => setTimeout(r, 0));
892 assert.equal(h.batcher.keysHeld, 0, 'acked batches release their keys');
893 assert.ok(h.batcher.add(key(4), 0, null));
894});
896test('doc reader: entries over the cap are cut (newest until first is kept); a text over the byte cap is refused', () => {
897 const entries = Array.from({ length: 10 }, (_, i) => [i.toString(16).padStart(KEY_HEX, '0'), 0, 2_000 - i, 1]);
898 const text = JSON.stringify({ v: 1, gen: 1, at: 1, entries });
899 const cut = indexDoc(text, { maxBytes: 1 << 20, maxEntries: 4 });
900 assert.equal(cut.entries, 4);
901 assert.equal(cut.truncated, true);
902 assert.ok(cut.index.has(`${'0'.repeat(KEY_HEX)}:0`));
903 assert.equal(indexDoc(text).truncated, false);
904 assert.throws(() => indexDoc(text, { maxBytes: text.length - 1, maxEntries: 4_096 }), /too large/);
905});
907test('doc reader: an oversized body is not read into the isolate (TOO_LARGE), and the last document stays in use', async () => {
908 resetRateState();
909 const ok = await serveDoc(new Request(DOC_URL), { blockedDoc: async () => ({ v: 1 as const, gen: 1, at: Date.now(), entries: [[K1, 0, Date.now() + 60_000, Date.now()] as const] }) });
910 const okText = await ok.text();
911 let big = false;
912 const ctx = {
913 waitUntil: () => {},
914 exports: {
915 RateCache: {
916 fetch: async () => {
917 if (!big) return new Response(okText, { headers: { etag: '"g1"' } });
918 // No content-length: a stream of 64 KiB chunks, far past MAX_DOC_BYTES. The reader must stop and cancel it.
919 let sent = 0;
920 const body = new ReadableStream<Uint8Array>({ pull(c) { sent++; c.enqueue(new Uint8Array(65_536).fill(32)); if (sent > 1_000) c.close(); } });
921 return new Response(body, { headers: { etag: '"g2"' } });
922 },
923 },
924 },
925 };
926 const first = await readDoc(ctx, Date.now());
927 assert.equal(first.read, 'ok');
928 big = true;
929 const second = await readDoc(ctx, Date.now());
930 assert.equal(second.read, 'memo', 'fell back to the last good document');
931 assert.equal(second.cache, 'TOO_LARGE');
932 assert.ok(second.memo !== null && second.memo.index.has(`${K1}:0`));
933 resetRateState();
934});
936test('core: the document query and the blocked prune use the until index (no scan + sort)', () => {
937 const db = new DatabaseSync(':memory:');
938 const exec: SqlExec = (q, ...b) => {
939 const stmt = db.prepare(q);
940 if (/^\s*select/i.test(q)) { const rows = stmt.all(...b) as Record<string, SqlValue>[]; return { toArray: () => rows, rowsWritten: 0 }; }
941 const r = stmt.run(...b);
942 return { toArray: () => [], rowsWritten: Number(r.changes) };
943 };
944 new CounterCore(exec);
945 const plan = (q: string) => (db.prepare(`EXPLAIN QUERY PLAN ${q}`).all(1, 4_096) as { detail: string }[]).map(r => r.detail).join(' | ');
946 const doc = plan('SELECT k, b, until, crossed FROM rate_blocked WHERE until > ? ORDER BY until DESC LIMIT ?');
947 assert.match(doc, /USING (COVERING )?INDEX rate_blocked_until/, doc);
948 assert.doesNotMatch(doc, /TEMP B-TREE/, doc);
949 const prune = plan('SELECT rowid AS r FROM rate_blocked WHERE until <= ? ORDER BY until LIMIT ?');
950 assert.match(prune, /USING (COVERING )?INDEX rate_blocked_until/, prune);
951});
953test('core: under a million-clients-a-minute inflow the prune takes out at least what each apply puts in', () => {
954 const core = new CounterCore(sqlite());
955 const t = 1_000_000_000_000;
956 let n = 0;
957 const fresh = (count: number) => Array.from({ length: count }, () => [(n++).toString(16).padStart(KEY_HEX, '0'), 0, 1] as const);
958 // A minute of full batches (2,048 distinct clients each): 30 applies, 61,440 hit rows.
959 for (let i = 0; i < 30; i++) core.apply(req(`m1:${i}`, fresh(2_048)), t + i * 2_000);
960 // Two minutes later all of it is expired, and the inflow goes on at the same rate.
961 let now = t + 3 * 60_000;
962 let backlog = core.backlog(now).hits;
963 assert.equal(backlog, 61_440);
964 let maxRows = 0;
965 for (let i = 0; i < 40; i++) {
966 const r = core.apply(req(`m3:${i}`, fresh(2_048)), now);
967 maxRows = Math.max(maxRows, r.rowsWritten);
968 assert.ok(r.pruned.hits >= Math.min(2_048, backlog), `apply ${i} pruned ${r.pruned.hits} of backlog ${backlog}`);
969 const next = core.backlog(now).hits;
970 assert.ok(next <= Math.max(0, backlog - 2_048), 'the backlog shrinks by at least the inflow + 128 per apply');
971 backlog = next;
972 now += 50;
973 }
974 assert.equal(backlog, 0, 'drained while the inflow continued');
975 // Bounded: an apply writes its own entries, its batch row, and at most entries + 128 (+1, + new blocks) pruned per table.
976 assert.ok(maxRows <= 2 * 2_048 + 3 * PRUNE_BATCH_ROWS + 10, `max rows written ${maxRows}`);
977});
Diagnostics4 git calls · 270 ms
CallStatusSizeTime0270 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=76
git.ls_refs.count=2
git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020040 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=264
git.response.bytes=41349
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=526b305930b8ef382f2d7a1f9c0238730a585921
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=6
git.pkt.data=4
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=41349
git.pkt.payload.max=41254
git.sideband.band1.bytes=41253
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=61
git.pack.bytes=41253
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=ab60efef4b7243678c75de64a4386986ee885c1c
git.pack.full=61
Open pack map
fetch20014 KiB264 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=14386
git.response.cache=MISS
git.response.duration_ms=264
git.fetch.wants=f427836344e6b6916a99d4e7b44381fe3eba2fa3
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=14386
git.pkt.payload.max=14365
git.sideband.band1.bytes=14364
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply-web.git
Ref
refs/heads/main
Commit
526b305930b8ef382f2d7a1f9c0238730a585921
cf-ray
a5e2e4a0bd712947
Request
a851fbb9…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format