{ "$comment": [ "Provenance of every threshold in scripts/cd/cd.policy.json and ply-web's limits (src/limits.ts, src/viewer/limits.ts), issue #73.", "Each entry is either status 'calibrated' with source, query, window, date, n, percentile and headroom, or status 'PRIOR (uncalibrated)' with why.", "value must equal the threshold's current value: changing a threshold without updating its citation fails test/calibrate-provenance.test.ts.", "metric names the distribution that calibrates it (attribute, route or '*', quantile, headroom multiplier, band ratio); scripts/calibrate --check reads it.", "Key form: ':'; policy paths may use [*] for every array element, with value the array of element values." ], "minN": { "value": 30, "status": "PRIOR (uncalibrated)", "why": "Fewest samples in a route's group before scripts/calibrate --check judges a band; below it the result is insufficient, never drift. A round floor." }, "thresholds": { "policy:steps[*].percent": { "value": [ 10, 25, 50, 75, 100 ], "status": "PRIOR (uncalibrated)", "why": "Ramp schedule chosen by hand; no step-size study. Calibrate from the per-step detectable effect (scripts/cd/forecast.ts lines) once enough ramps are logged." }, "policy:steps[*].dwellMs": { "value": [ 30000, 25000, 25000, 25000, 40000 ], "status": "calibrated", "source": "ledger ramp-step observations of runs 2026-11-06T17-35-09-419Z through 2026-11-06T21-06-26-841Z; live-check durations of the promote runs 2026-11-06T21-45-51-484Z through 2026-11-07T19-39-00-604Z", "query": "ramp-step measurement nextRequests / aggregateRequests per step; run_finished stage durations smoke-live + telemetry-live", "window": "10/25/50/75% ramp steps (n=33); 100% promote watches (n=10)", "date": "2026-11-08", "n": 43, "percentile": "minimum observed new-version share per percent (0.0874 at 10% excluding the pre-warm-up 0.0712 rollback, 0.232 at 25%); median live-check time 49 s; not a percentile", "headroom": "10%: 1500 issued x 0.0874 = 131 new vs split.minNewRequests 100. 25-75%: 1250 issued, 25% above probe.minSamples 1000. 100%: unchanged, bounded by the live checks. Full derivation: cd.policy.json calibration[steps[*].dwellMs]." }, "policy:ingestWaitMs": { "value": 20000, "status": "calibrated", "source": "runs 2026-11-08T16-58-16-960Z and 2026-11-08T17-07-46-259Z (polled ingest); run 2026-11-01T00-10-02-620Z", "query": "ramp-step aggregateRequests per poll after the dwell end", "window": "every ramp step of both 2026-11-08 runs", "date": "2026-11-08", "n": 10, "percentile": "logged count plateaued at the 4th poll, 21.8-24.1 s after the dwell end, on all 5 steps of 2026-11-08T17-07-46-259Z (n=5); the ~93% plateau was the $metadata.id dedupe bug (fixed ace0eb9), not ingest; not a percentile", "headroom": "the last poll under a 20000 ceiling returns at about +22-24 s, at the measured plateau; 30000 added two flat polls a step (ramp 374 s vs 325 s)" }, "policy:warmupMs": { "value": 10000, "status": "calibrated", "source": "runs 2026-11-06T20-01-37-842Z and 2026-11-06T20-11-49-685Z", "query": "old-version request events after the 100% deployment's created_on, Workers Observability events view, service ply, partitioned by $workers.scriptVersion.id", "window": "100% ramp steps of 86456068 (previous c43f01cd)", "date": "2026-11-06", "n": 2, "percentile": "max of two measured split-propagation tails (4.6 s, 3.7 s; 24 and 25 old-version requests, none later); not a percentile", "headroom": "10000 is 2.2x the larger tail" }, "policy:probe.rps": { "value": 25, "status": "PRIOR (uncalibrated)", "why": "Round probe rate; it sets the probe sample count per dwell (rps x calls x dwell), not derived from a distribution." }, "policy:probe.maxLossRate": { "value": 0.0012658227848101266, "status": "calibrated", "source": "run 2026-11-01T17-06-08-780Z", "query": "probe outcomes (scripts/cd/probe.ts) lost / issued", "window": "100% ramp step", "date": "2026-11-01", "n": 790, "percentile": "one measured step (1/790), not a percentile", "headroom": "none above the measured loss; a greater rate is a fault (cd.policy.json probe.calibration)" }, "policy:probe.timeoutMs": { "value": 2000, "status": "calibrated", "source": "run 2026-11-01T19-40-35-485Z", "query": "probe latencyMs by outcome (scripts/cd/probe.ts)", "window": "50% ramp step", "date": "2026-11-01", "n": 118, "percentile": "healthy response p50 41-101ms on both versions", "headroom": "above 10x the healthy p50 upper bound (1010ms), below the 69830ms stall (cd.policy.json probe.schedule)" }, "policy:probe.maxInFlight": { "value": 50, "status": "PRIOR (uncalibrated)", "why": "cd.policy.json probe.schedule.headroom: the previous cap was one second of ticks times 2 calls; healthy in-flight is about 5." }, "policy:probe.minSamples": { "value": 1000, "status": "PRIOR (uncalibrated)", "why": "cd.policy.json probe.schedule.headroom: a round floor above the 128-sample stalled window and below the ~1990 of a healthy step." }, "policy:probe.stallLatencyMs": { "value": 1000, "status": "PRIOR (uncalibrated)", "why": "cd.policy.json probe.schedule.headroom: a round threshold above the 101ms p50 and below timeoutMs." }, "policy:probe.windowAttempts": { "value": 2, "status": "calibrated", "source": "run 2026-11-01T19-40-35-485Z", "query": "stalled probe windows per step (scripts/cd/probe.ts judgeProbeWindow retry)", "window": "50% ramp step", "date": "2026-11-01", "n": 1, "percentile": "one stalled window observed, not a percentile", "headroom": "the stalled window plus one re-run" }, "policy:probe.windowBackoffMs": { "value": 1000, "status": "PRIOR (uncalibrated)", "why": "cd.policy.json probe.schedule.headroom: the run did not wait between windows." }, "policy:burn.fastBurn": { "value": 14.4, "status": "PRIOR (uncalibrated)", "why": "Google SRE workbook fast-burn multiple (scripts/cd/slo.ts FAST_BURN): an external convention, not a ply-web distribution." }, "policy:burn.minSamples": { "value": 100, "status": "PRIOR (uncalibrated)", "why": "Round floor before a burn rate is judged (scripts/cd/slo.ts MIN_SAMPLES)." }, "policy:split.z": { "value": 3, "status": "PRIOR (uncalibrated)", "why": "Three standard errors on the routed-share binomial band; a convention, not fitted to observed split noise." }, "policy:split.minNewRequests": { "value": 100, "status": "PRIOR (uncalibrated)", "why": "Round floor of new-version requests before a step can advance." }, "policy:telemetry.maxLossRate": { "value": 0.5, "status": "PRIOR (uncalibrated)", "why": "Default (scripts/cd/policy.ts DEFAULT_MAX_LOSS_RATE): fail only when more than half the smoke ids are missing. Calibrate from the telemetry-loss observations the CD logs per run." }, "policy:telemetry.query.attempts": { "value": 3, "status": "calibrated", "source": "run 2026-11-01T19-17-18-170Z", "query": "cf o11y telemetry query exit status per invocation (scripts/cd/query-retry.ts)", "window": "100% ramp step", "date": "2026-11-01", "n": 1, "percentile": "one measured timeout, not a percentile", "headroom": "two attempts past the single timeout that rolled the step back" }, "policy:telemetry.query.backoffMs": { "value": 1000, "status": "PRIOR (uncalibrated)", "why": "cd.policy.json telemetry.query.calibration.headroom: the run did not retry." }, "policy:cleanup.attempts": { "value": 3, "status": "calibrated", "source": "run 2026-11-01T17-29-43-122Z", "query": "cf artifacts repo delete exit status (scripts/cd/cleanup.ts)", "window": "post-ramp scratch repo delete", "date": "2026-11-01", "n": 1, "percentile": "one measured delete (103000ms timeout, then 404), not a percentile", "headroom": "one attempt past the timeout-then-404 that confirmed deletion" }, "policy:cleanup.backoffMs": { "value": 1000, "status": "PRIOR (uncalibrated)", "why": "cd.policy.json cleanup.calibration.headroom: the confirming retry was immediate." }, "policy:cleanup.staleMinutes": { "value": 60, "status": "PRIOR (uncalibrated)", "why": "cd.policy.json cleanup.calibration.headroom: the previous sweep age, not a measured leak age." }, "api:maxObjectSize": { "value": 16777216, "status": "PRIOR (uncalibrated)", "why": "Derived from the 128 MiB isolate budget (src/limits.ts), not from a request distribution.", "metric": { "attribute": "git.read.blob.size", "route": "api", "quantile": 0.999, "headroom": 4, "band": 2 } }, "api:maxPackBytes": { "value": 33554432, "status": "PRIOR (uncalibrated)", "why": "Derived from the 128 MiB isolate budget (src/limits.ts), not from a request distribution.", "metric": { "attribute": "git.pack.bytes", "route": "api", "quantile": 0.999, "headroom": 4, "band": 2 } }, "api:maxHarvestBytes": { "value": 16777216, "status": "PRIOR (uncalibrated)", "why": "Derived from the 128 MiB isolate budget (src/limits.ts), not from a request distribution." }, "api:maxInflatedTotal": { "value": 67108864, "status": "PRIOR (uncalibrated)", "why": "Four maximum-sized objects (merge-base triple plus commit) under the isolate budget; not from a distribution.", "metric": { "attribute": "git.pack.inflated.bytes", "route": "api", "quantile": 0.999, "headroom": 4, "band": 2 } }, "api:maxPackObjects": { "value": 100000, "status": "PRIOR (uncalibrated)", "why": "ply's buffered-pack object ceiling; the inflated-byte cap is the memory bound.", "metric": { "attribute": "git.pack.objects", "route": "api", "quantile": 0.999, "headroom": 4, "band": 2 } }, "api:maxLsRefsBytes": { "value": 33554432, "status": "PRIOR (uncalibrated)", "why": "Shares the pack byte cap: one ref listing is buffered.", "metric": { "attribute": "git.response.bytes", "route": "GET /v1/refs", "quantile": 0.999, "headroom": 4, "band": 2 } }, "api:requestTimeoutMs": { "value": 10000, "status": "calibrated", "source": "loop 2026-11-08 phase6 remeasure + Workers Observability calibrate (service ply); cited in ivarvong/ply-web#73", "query": "ply-loadgen phase6 worker/* latencies (identical suite) and git.response.duration_ms / request wall times on ply @ fcb0148c", "window": "phase6 remeasure 2026-11-08T09:20 America/New_York; calibrate 30m post-promote", "date": "2026-11-08", "n": 20, "percentile": "phase6 max on slowest route worker/read/big20m = 2624ms; other routes max \u2264652ms; not near 10s", "headroom": "10000 is ~3.8x the observed max (2624ms); prior 30000 was PRIOR (uncalibrated)", "comment": "https://github.com/ivarvong/ply-web/issues/73", "metric": { "attribute": "http.request.duration_ms", "route": "GET /v1/blob (read/big20m)", "quantile": 1.0, "headroom": 3.8, "band": 2 } }, "api:maxBodyBytes": { "value": 1048576, "status": "PRIOR (uncalibrated)", "why": "Round JSON request body cap." }, "mirror:maxPackBytes": { "value": 33554432, "status": "PRIOR (uncalibrated)", "why": "Mirror cron sizes (src/limits.ts MIRROR_CAPS) chosen for the two mirrored repos; no sync-size distribution yet." }, "mirror:maxObjectSize": { "value": 16777216, "status": "PRIOR (uncalibrated)", "why": "Mirror cron sizes (src/limits.ts MIRROR_CAPS) chosen for the two mirrored repos; no sync-size distribution yet." }, "mirror:maxInflatedTotal": { "value": 100663296, "status": "PRIOR (uncalibrated)", "why": "Mirror cron sizes (src/limits.ts MIRROR_CAPS) chosen for the two mirrored repos; no sync-size distribution yet." }, "mirror:maxHarvestBytes": { "value": 16777216, "status": "PRIOR (uncalibrated)", "why": "Mirror cron sizes (src/limits.ts MIRROR_CAPS) chosen for the two mirrored repos; no sync-size distribution yet." }, "mirror:maxPackObjects": { "value": 50000, "status": "PRIOR (uncalibrated)", "why": "Mirror cron sizes (src/limits.ts MIRROR_CAPS) chosen for the two mirrored repos; no sync-size distribution yet." }, "mirror:requestTimeoutMs": { "value": 30000, "status": "PRIOR (uncalibrated)", "why": "Mirror cron sizes (src/limits.ts MIRROR_CAPS) chosen for the two mirrored repos; no sync-size distribution yet." }, "verify:maxPackBytes": { "value": 16777216, "status": "PRIOR (uncalibrated)", "why": "GitCache verification buffers the pack (src/limits.ts VERIFY_CAPS); sized to the isolate, not to a cached-pack distribution.", "metric": { "attribute": "git.pack.bytes", "route": "*", "quantile": 0.999, "headroom": 4, "band": 2 } }, "verify:maxObjectSize": { "value": 16777216, "status": "PRIOR (uncalibrated)", "why": "GitCache verification buffers the pack (src/limits.ts VERIFY_CAPS); sized to the isolate, not to a cached-pack distribution." }, "verify:maxInflatedTotal": { "value": 67108864, "status": "PRIOR (uncalibrated)", "why": "GitCache verification buffers the pack (src/limits.ts VERIFY_CAPS); sized to the isolate, not to a cached-pack distribution." }, "verify:maxHarvestBytes": { "value": 16777216, "status": "PRIOR (uncalibrated)", "why": "GitCache verification buffers the pack (src/limits.ts VERIFY_CAPS); sized to the isolate, not to a cached-pack distribution." }, "verify:maxPackObjects": { "value": 100000, "status": "PRIOR (uncalibrated)", "why": "GitCache verification buffers the pack (src/limits.ts VERIFY_CAPS); sized to the isolate, not to a cached-pack distribution." }, "verify:maxTreeEntries": { "value": 1000000, "status": "PRIOR (uncalibrated)", "why": "Verification must not refuse a tree the reader accepted; set far above any read cap." }, "verify:requestTimeoutMs": { "value": 30000, "status": "PRIOR (uncalibrated)", "why": "GitCache verification buffers the pack (src/limits.ts VERIFY_CAPS); sized to the isolate, not to a cached-pack distribution." }, "viewer:maxUpstreamBytes": { "value": 4194304, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxPackBytes": { "value": 2097152, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution.", "metric": { "attribute": "git.pack.bytes", "route": "viewer", "quantile": 0.999, "headroom": 4, "band": 2 } }, "viewer:maxPackObjects": { "value": 400, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution.", "metric": { "attribute": "git.pack.objects", "route": "viewer", "quantile": 0.999, "headroom": 4, "band": 2 } }, "viewer:maxInflatedTotal": { "value": 4194304, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution.", "metric": { "attribute": "git.pack.inflated.bytes", "route": "viewer", "quantile": 0.999, "headroom": 4, "band": 2 } }, "viewer:maxObjectSize": { "value": 1048576, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxFileDisplayBytes": { "value": 262144, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxDeltaChain": { "value": 32, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution.", "metric": { "attribute": "git.pack.chain.max", "route": "viewer", "quantile": 0.999, "headroom": 4, "band": 2 } }, "viewer:maxDeltaResult": { "value": 1048576, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:requestTimeoutMs": { "value": 12000, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution.", "metric": { "attribute": "git.response.duration_ms", "route": "viewer", "quantile": 0.999, "headroom": 10, "band": 2 } }, "viewer:pageTimeoutMs": { "value": 20000, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxSubrequests": { "value": 16, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxRedirects": { "value": 2, "status": "PRIOR (uncalibrated)", "why": "Protocol-safety bound, not a load threshold." }, "viewer:maxRefs": { "value": 2000, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution.", "metric": { "attribute": "git.ls_refs.count", "route": "viewer", "quantile": 0.999, "headroom": 4, "band": 2 } }, "viewer:maxBranchesShown": { "value": 200, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxTagsShown": { "value": 100, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:hintNearbyRefs": { "value": 4, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxTreeEntries": { "value": 2000, "status": "PRIOR (uncalibrated)", "why": "Back to 2,000 after ply #47 (src/viewer/limits.ts); a design value, not from a tree-size distribution." }, "viewer:maxTreeShown": { "value": 500, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxLogCommits": { "value": 10, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxPreviewChars": { "value": 2000, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxUrlLength": { "value": 2048, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:maxLineRows": { "value": 4000, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:ratePerMinute": { "value": 60, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:packRatePerMinute": { "value": 20, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:rateWindowMs": { "value": 60000, "status": "PRIOR (uncalibrated)", "why": "Public viewer cap chosen by hand (src/viewer/limits.ts); not from a viewer request distribution." }, "viewer:rateCeiling": { "value": 1000, "status": "PRIOR (uncalibrated)", "why": "Clamp on an operator-set rate, not a load threshold." }, "policy:ingestPollMs": { "value": 3000, "status": "PRIOR (uncalibrated)", "why": "Re-query period for the polled ingest wait after each ramp step (moved from promote.ingestPollMs 2026-11-08, when the direct promote was removed): each poll is two telemetry queries of ~1 s (run 2026-11-06T21-06-26-841Z), so 3 s keeps the query load under one in flight. policy:ingestWaitMs stays the ceiling; the poll stops early only once >= 98% of answered probes are logged (scripts/cd/ramp.ts INGEST_LANDED)." }, "policy:rate.probes": { "value": 30, "status": "PRIOR (uncalibrated)", "why": "Round probe count for the rate gate (scripts/cd/rate-gate.ts): enough warm default-mode checks to judge a 10% fail_open share at a 0% canary, which has no other viewer traffic." }, "policy:rate.probeGapMs": { "value": 300, "status": "PRIOR (uncalibrated)", "why": "Above a fresh isolate's first RateCache read (MISS 403-481 ms on lat2 is the worst; HIT/SWR p99 12 ms) so a probe rarely lands while its isolate's first read is still in flight; 30 probes take ~9 s." }, "policy:rate.minChecks": { "value": 20, "status": "PRIOR (uncalibrated)", "why": "Two thirds of the 30 probes: tolerates a third lost to ingest before the fail_open share is unevaluable (a harness fault, not a pass)." }, "policy:rate.maxFailOpenRate": { "value": 0.1, "status": "calibrated", "source": "Workers Observability spans, 7 days to 2026-11-07T17:30Z, every version since the batched limiter shipped (6ed2e746 through e8846911; runs lat1-lat8, abuse1-5, fault1-2)", "query": "count of ply.rate.check by ply.script_version, rate.mode, rate.decision (calculations view), split by rate.load", "window": "7 days, per version", "date": "2026-11-07", "n": 151822, "percentile": "share, not a percentile: all batched checks 156/151822 = 0.10% fail_open; load runs per version 0-0.13% (2a4741c0 109/83,574); real (non-load) traffic per version 36-566 checks, worst 7.7% (ccad274e 4/52, cold isolates included)", "headroom": "0.1 is above the worst real-traffic share (7.7%, which included the cold-isolate first reads the gate excludes) and 75x the load share (0.13%); a broken read path fails open ~100%" }, "policy:rate.maxDroppedFlushes": { "value": 0, "status": "calibrated", "source": "Workers Observability spans, 7 days to 2026-11-07T17:30Z, every version since the batched limiter shipped (6ed2e746 through e8846911; runs lat1-lat8, abuse1-5, fault1-2)", "query": "count of ply.rate.flush by ply.script_version, rate.batch.outcome, rate.batch.deadline_hit; dropped ones by rate.fault; ply.rate.check with rate.batch.dropped", "window": "7 days, 11 versions", "date": "2026-11-07", "n": 128753, "percentile": "count, not a percentile: 1 dropped of 128,753 and it carried an injected fault (a020b1b1 error;attempts=2, run fault2); 0 overflow", "headroom": "none: the no-fault baseline is 0 (rule of three: < 2.3e-5 per flush), so any drop without a fault fails" } } }