name: nightly # Deep fuzz against main with a seed that rotates daily (the UTC date as YYYYMMDD), the full `npm test`, the full # limit sweep, and the pack-cut space with git's oracle on every cut (PACK_SPACE=full). No Stryker: mutate.yml runs the # full src/ mutation run, in 16 shards, on every push to main, and a mutation run of unchanged code is the same run # (unsharded it is ~8 h of test time and never fit this job's 300 min: nightly 1, 4 and 5 were cancelled at it). # A failure prints the seed, the failing case and a one-line reproduce command (NIGHTLY-FAIL lines), and the report # job opens or comments on the issue "nightly fuzz failure ". on: schedule: - cron: "23 7 * * *" workflow_dispatch: inputs: seed: description: "fuzz seed (digits); empty = today's UTC date as YYYYMMDD" required: false default: "" permissions: {} concurrency: group: nightly cancel-in-progress: false jobs: seed: name: seed runs-on: ubuntu-latest permissions: {} outputs: seed: ${{ steps.seed.outputs.seed }} date: ${{ steps.seed.outputs.date }} sha: ${{ steps.seed.outputs.sha }} steps: - id: seed env: SEED_INPUT: ${{ inputs.seed }} run: | set -euo pipefail date=$(date -u +%Y-%m-%d) seed="${SEED_INPUT:-$(date -u +%Y%m%d)}" [[ "$seed" =~ ^[0-9]{1,9}$ ]] || { echo "::error::seed must be 1-9 digits, got '$seed'"; exit 1; } echo "seed=$seed" >> "$GITHUB_OUTPUT" echo "date=$date" >> "$GITHUB_OUTPUT" # One commit for the whole run. Every job checks out this sha, never `main`: the shard plan is a pure # function of one commit's file list and test/durations.json, so shards that each resolved main on # their own could run a file twice or not at all when main moved between their checkouts, and gate c # would judge records from mixed commits. A re-run of failed jobs keeps this output, so it stays pinned. sha=$(git ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" refs/heads/main | cut -f1) [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::could not resolve main to one commit, got '$sha'"; exit 1; } echo "sha=$sha" >> "$GITHUB_OUTPUT" echo "nightly seed $seed ($date) at $sha" fuzz: name: fuzz ${{ matrix.fuzzer }} ${{ matrix.args }} needs: seed runs-on: ubuntu-latest timeout-minutes: 120 permissions: contents: read strategy: fail-fast: false matrix: include: - { fuzzer: run.mjs, args: "--n 1000" } - { fuzzer: hostile.mjs, args: "--n 3000" } - { fuzzer: liar.mjs, args: "--n 2000" } - { fuzzer: protocol.mjs, args: "--n 5000" } - { fuzzer: packs.mjs, args: "--n 1000" } - { fuzzer: packgen.mjs, args: "--n 2000" } - { fuzzer: codec.mjs, args: "--n 300000" } - { fuzzer: attributes.mjs, args: "--n 2000" } - { fuzzer: below-hash.mjs, args: "--n 2000" } - { fuzzer: git-semantic.mjs, args: "--n 5000" } - { fuzzer: linemerge.mjs, args: "--n 20000" } - { fuzzer: linemerge.mjs, args: "--n 5000 --runs" } - { fuzzer: linemerge.mjs, args: "--n 5000 --driver union" } - { fuzzer: diffscript.mjs, args: "--n 5000" } - { fuzzer: diffscript.mjs, args: "--n 2000 --snake" } - { fuzzer: diffscript.mjs, args: "--n 200 --huge" } env: GIT_AUTHOR_NAME: ply GIT_AUTHOR_EMAIL: ply@example.com GIT_COMMITTER_NAME: ply GIT_COMMITTER_EMAIL: ply@example.com steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ needs.seed.outputs.sha }} persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 - name: git identity run: git config --global user.name ply && git config --global user.email ply@example.com - run: npm ci - name: fuzz env: SEED: ${{ needs.seed.outputs.seed }} FUZZER: ${{ matrix.fuzzer }} ARGS: ${{ matrix.args }} run: | # ARGS is word-split on purpose: it is a fixed list from the matrix above # shellcheck disable=SC2086 node scripts/nightly-fuzz.mjs "$SEED" "$FUZZER" $ARGS # `npm test`, split across 4 runners the way ci.yml's test-shard is (the weighted sequencer in vitest.config.ts over # test/durations.json): on one runner every heavy file ran beside the others, and the push-outcome boundary space # passed its 30 min bound there (#263). Job `test` then judges gate c over every shard's records and runs the # pinned fuzz lines, which is the rest of `npm test`. test-shard: name: npm test shard (${{ matrix.shard }}) needs: seed runs-on: ubuntu-latest timeout-minutes: 90 permissions: contents: read strategy: fail-fast: false matrix: shard: [1, 2, 3, 4] env: GIT_AUTHOR_NAME: ply GIT_AUTHOR_EMAIL: ply@example.com GIT_COMMITTER_NAME: ply GIT_COMMITTER_EMAIL: ply@example.com steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ needs.seed.outputs.sha }} persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 - name: git identity run: git config --global user.name ply && git config --global user.email ply@example.com - run: npm ci - name: vitest env: INDEX: ${{ strategy.job-index }} TOTAL: ${{ strategy.job-total }} PLY_FAILSITES: .failsites NODE_OPTIONS: --import ${{ github.workspace }}/test/space/failsite-hook.mjs run: | shard="$((INDEX + 1))/${TOTAL}" npx vitest run --shard "$shard" || { echo "NIGHTLY-FAIL npm-test shard $shard reproduce: npx vitest run --shard $shard (the failing files are the FAIL lines above)"; exit 1; } - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: success() || failure() with: name: failsites-${{ matrix.shard }} path: .failsites include-hidden-files: true if-no-files-found: error retention-days: 14 test: name: npm test needs: [seed, test-shard] if: always() runs-on: ubuntu-latest timeout-minutes: 60 permissions: contents: read actions: read # the shards' fail-site records env: GIT_AUTHOR_NAME: ply GIT_AUTHOR_EMAIL: ply@example.com GIT_COMMITTER_NAME: ply GIT_COMMITTER_EMAIL: ply@example.com steps: - name: every shard succeeded env: SHARDS: ${{ needs.test-shard.result }} run: | if [ "$SHARDS" != success ]; then echo "NIGHTLY-FAIL npm-test shards: $SHARDS (see the npm test shard jobs)" exit 1 fi - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ needs.seed.outputs.sha }} persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 - name: git identity run: git config --global user.name ply && git config --global user.email ply@example.com - run: npm ci - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: failsites-* path: .failsites merge-multiple: true - name: gate c (fail sites) env: PLY_FAILSITES: .failsites PLY_FAILSITES_GATE: 1 run: | npx vitest run test/space/failsites.test.js || { echo "NIGHTLY-FAIL gate-c reproduce: npm test (gate c runs after the full vitest run)"; exit 1; } - name: pinned fuzz lines run: | # the gate's list: every seed and case is a constant in test/fuzz/pinned.mjs, which prints # `FUZZ-PINNED FAIL seed=` and a `replay:` command on a failure or a hang npm run test:fuzz-pinned || { echo "NIGHTLY-FAIL npm-test fuzz reproduce: npm run test:fuzz-pinned (see the FUZZ-PINNED FAIL and replay: lines above)"; exit 1; } - name: pinned fuzz lines, fresh seed (discovery) run: | # the same list with one seed drawn for this run; the first line printed is the seed and # `replay: node test/fuzz/pinned.mjs --seed `, and a failure repeats it npm run test:fuzz-random || { echo "NIGHTLY-FAIL fuzz-random reproduce: node test/fuzz/pinned.mjs --seed (the seed is on the 'fuzz-pinned: random mode' lines above)"; exit 1; } pack-cut-full: # every ply path already runs every cut on each PR; this adds git index-pack --strict on every cut (~9,600 runs, # ~90 s), where the PR run judges field boundaries and every 8th offset (~1,400 runs) name: pack-cut (git oracle on every cut) needs: seed runs-on: ubuntu-latest timeout-minutes: 30 permissions: contents: read steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ needs.seed.outputs.sha }} persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 - name: git identity run: git config --global user.name ply && git config --global user.email ply@example.com - run: npm ci - name: pack-cut, full oracle env: PACK_SPACE: full run: | npx vitest run test/space/pack-cut.test.js || { echo "NIGHTLY-FAIL pack-cut reproduce: PACK_SPACE=full npx vitest run test/space/pack-cut.test.js"; exit 1; } sweep: name: limit sweep (full) needs: seed runs-on: ubuntu-latest timeout-minutes: 180 permissions: contents: read env: GIT_AUTHOR_NAME: ply GIT_AUTHOR_EMAIL: ply@example.com GIT_COMMITTER_NAME: ply GIT_COMMITTER_EMAIL: ply@example.com steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ needs.seed.outputs.sha }} persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 - name: git identity run: git config --global user.name ply && git config --global user.email ply@example.com - run: npm ci - name: sweep (full matrix) run: | node --import ./test/ts-resolve.mjs test/sweep/run.mjs --full || { echo "NIGHTLY-FAIL sweep reproduce: npm run sweep -- --full"; exit 1; } report: name: report failure needs: [seed, fuzz, test-shard, test, pack-cut-full, sweep] # a job that hits its timeout-minutes ends cancelled, not failed: report that too, or a timeout is silent if: failure() || cancelled() runs-on: ubuntu-latest permissions: issues: write actions: read # the failed jobs' logs, for their NIGHTLY-FAIL lines steps: - name: open or update the issue env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} SEED: ${{ needs.seed.outputs.seed }} DATE: ${{ needs.seed.outputs.date }} SHA: ${{ needs.seed.outputs.sha }} run: | set -euo pipefail date="${DATE:-$(date -u +%Y-%m-%d)}" title="nightly fuzz failure $date" body=$(mktemp) { echo "Nightly run $RUN_URL failed or timed out on main at ${SHA:-unknown} (seed \`${SEED:-unknown}\`)." echo gh api --paginate "repos/$GH_REPO/actions/runs/$RUN_ID/jobs" --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "cancelled") | "\(.id)\t\(.name)"' | while IFS=$'\t' read -r id name; do echo "### $name" echo '```' # A job's log is not served for a while after the job ends: this step started 9 s after npm test # failed on 2026-11-10 and got nothing, so the issue had an empty excerpt (#263). Retry until the log # carries its NIGHTLY-FAIL line, and say so when it never does instead of posting an empty block. log="" for _ in 1 2 3 4 5 6 7 8; do log=$(gh api "repos/$GH_REPO/actions/jobs/$id/logs" 2>/dev/null || true) if printf '%s\n' "$log" | grep -qE '^[0-9TZ:.-]+ NIGHTLY-FAIL '; then break; fi sleep 15 done # a log line is " ": anchoring skips the echoed script and the ##[error] copies; # vitest's FAIL and timeout lines name the failing file and test (colour codes stripped) excerpt=$(printf '%s\n' "$log" | sed -E 's/\x1b\[[0-9;]*m//g' | grep -E '^[0-9TZ:.-]+ (NIGHTLY-FAIL | FAIL |Error: Test timed out|##\[error\]The job running on runner .* has exceeded)' | cut -d' ' -f2- | awk '!seen[$0]++' | head -20 || true) if [ -n "$excerpt" ]; then printf '%s\n' "$excerpt" elif [ -z "$log" ]; then echo "(job log not available after 8 tries over 2 min; open the run above)" else echo "(no NIGHTLY-FAIL or FAIL line in the job log: it was likely cancelled or timed out; open the run above)" fi echo '```' done } > "$body" cat "$body" num=$(gh issue list --state open --search "\"$title\" in:title" --json number,title --jq ".[] | select(.title == \"$title\") | .number" | head -1) if [ -n "$num" ]; then gh issue comment "$num" --body-file "$body" else gh issue create --title "$title" --body-file "$body" fi