fix(skills): sync-baseline --dry-run + pr-resolve/gate-c/review-req correctness (F30, F32, F4b, F13) #329
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark gate | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| jobs: | |
| benchmark-solve: | |
| name: Solve smoke benchmark | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - name: Install dependencies | |
| run: npm ci --ignore-scripts || npm install --ignore-scripts | |
| - name: Build artifacts | |
| run: npm run build:hooks && npm run build:machines | |
| - name: Run solve smoke benchmark | |
| id: bench | |
| run: | | |
| node bin/nf-benchmark-solve.cjs --json --track=smoke > bench-solve-output.json 2>&1 || true | |
| cat bench-solve-output.json | |
| - name: Check score against baseline | |
| run: node scripts/check-benchmark-gate.cjs bench-solve-output.json benchmarks/solve-baseline.json | |
| benchmark-quick: | |
| name: Quick smoke benchmark | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| needs: benchmark-solve | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - name: Install dependencies | |
| run: npm ci --ignore-scripts || npm install --ignore-scripts | |
| - name: Build artifacts | |
| run: npm run build:hooks && npm run build:machines | |
| - name: Run quick smoke benchmark | |
| run: | | |
| node bin/nf-benchmark.cjs --skill=quick --track=smoke --json > bench-quick-output.json 2>&1 || true | |
| cat bench-quick-output.json | |
| - name: Check score against baseline | |
| run: node scripts/check-benchmark-gate.cjs bench-quick-output.json benchmarks/quick/baseline.json | |
| # ── Debug benchmark: structural check (no API key, ~10s) ────────────────── | |
| # Runs on every PR. Verifies all 100 stubs still have bugs, all 100 tests | |
| # are syntactically valid, no hint comments crept back in, and every assert | |
| # helper uses the counter form. Catches accidental stub/test corruption | |
| # without any LLM invocation. | |
| benchmark-debug-structure: | |
| name: Debug benchmark — structural check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| needs: benchmark-quick | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - name: Install dependencies | |
| run: npm ci --ignore-scripts || npm install --ignore-scripts | |
| - name: Structural integrity check | |
| run: node bin/check-debug-bench-structure.cjs | |
| # ── Debug benchmark: smoke LLM check (needs ANTHROPIC_API_KEY, ~5 min) ─── | |
| # Runs 5 easy stubs through the full fix-cycle pipeline on every PR. | |
| # Verifies the debug runner, Claude CLI integration, and fix extraction all | |
| # work end-to-end. Score must be >= smoke baseline (5/5 = 100%). | |
| # Skipped automatically if ANTHROPIC_API_KEY is absent (fork PRs, etc.). | |
| benchmark-debug-smoke: | |
| name: Debug benchmark — smoke LLM (5 stubs) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: benchmark-debug-structure | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - name: Install dependencies | |
| run: npm ci --ignore-scripts || npm install --ignore-scripts | |
| - name: Build artifacts | |
| run: npm run build:hooks && npm run build:machines | |
| - name: Run debug smoke benchmark (5 easy stubs) | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: | | |
| if [ -z "$ANTHROPIC_API_KEY" ]; then | |
| echo "ANTHROPIC_API_KEY not set — skipping smoke LLM check" | |
| echo '{"score":100,"pass_rate":100,"total":5,"fixed":5,"passed":5,"track":"smoke","skipped":true}' > bench-debug-smoke.json | |
| else | |
| node bin/nf-benchmark-debug.cjs --json --track smoke > bench-debug-smoke.json 2>&1 || true | |
| fi | |
| cat bench-debug-smoke.json | |
| - name: Check score against smoke baseline | |
| run: node scripts/check-benchmark-gate.cjs bench-debug-smoke.json benchmarks/debug/smoke-baseline.json |