benchmark.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: "JMH Benchmark Comparison"

# SECURITY: This workflow deliberately uses pull_request, never pull_request_target.
# Pull requests run untrusted code, and Apache Infra policy forbids exposing tokens
# to that code through a privileged pull_request_target workflow.
on:
  pull_request:
    types: [opened, synchronize, reopened, labeled]
    paths-ignore:
      - '**/*.md'
      - '**/*.adoc'
      - 'grails-doc/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.event.action == 'labeled' && github.event.label.name != 'performance' && github.run_id || 'benchmark' }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read

jobs:
  # Each shard builds both revisions before measuring either one on the same runner.
  # Building first avoids CPU, IO, thermal, cache, and frequency state biasing a measurement.
  # Shard a measures BASE then HEAD, while shard b measures HEAD then BASE.
  # Alternating the order cancels first-versus-second ordering bias without doubling runtime.
  # Both shards use the PR merge commit as HEAD, so they measure what would actually land.
  # Results are advisory: a detected regression never fails this workflow.
  benchmark:
    name: "Paired JMH benchmarks (${{ matrix.shard }})"
    if: >-
      contains(github.event.pull_request.labels.*.name, 'performance') &&
      (github.event.action != 'labeled' || github.event.label.name == 'performance')
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        shard: [a, b]
    env:
      BASE_SHA: ${{ github.event.pull_request.base.sha }}
      HEAD_SHA: ${{ github.sha }}
      JMH_INCLUDE: '.*'
      RESULT_DIR: ${{ github.workspace }}/jmh-results/${{ matrix.shard }}
      REPORT_DIR: ${{ github.workspace }}/jmh-reports/${{ matrix.shard }}
      SHARD: ${{ matrix.shard }}
    steps:
      - name: "📥 Checkout repository"
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
      - name: "☕️ Setup JDK"
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
        with:
          distribution: liberica
          java-version: 21
      - name: "🐘 Setup Gradle"
        uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
        with:
          cache-provider: basic # 'basic' uses the MIT-licensed, open-source cache provider; the default 'enhanced' provider (v6+) is proprietary (Gradle commercial Terms of Use)
          develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      # Deliberately allowed to fail the job, and placed before the expensive work: a broken
      # comparison tool makes every number produced here untrustworthy. That is a tooling
      # failure rather than a performance finding, so regressions themselves stay advisory.
      - name: "🧪 Verify JMH comparison tool"
        run: ./gradlew :grails-benchmarks:test --max-workers=4
      - name: "🌳 Prepare paired worktrees"
        run: |
          WORKTREE_ROOT="$RUNNER_TEMP/jmh-worktrees/$SHARD"
          mkdir -p "$WORKTREE_ROOT" "$RESULT_DIR" "$REPORT_DIR"
          if [ -n "$BASE_SHA" ] && git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then
            RESOLVED_BASE_SHA="$BASE_SHA"
          elif RESOLVED_BASE_SHA="$(git merge-base "$HEAD_SHA^1" "$HEAD_SHA^2" 2>/dev/null)"; then
            echo "Configured base commit is unreachable; using merge-base $RESOLVED_BASE_SHA."
          elif RESOLVED_BASE_SHA="$(git rev-parse "$HEAD_SHA^" 2>/dev/null)"; then
            echo "Configured base commit is unreachable; using HEAD parent $RESOLVED_BASE_SHA."
          else
            RESOLVED_BASE_SHA=""
            echo "No base commit could be resolved; comparison will use HEAD-only mode."
          fi
          git worktree add --detach "$WORKTREE_ROOT/head" "$HEAD_SHA"
          {
            printf 'HEAD_DIR=%s\n' "$WORKTREE_ROOT/head"
            printf 'WORKTREE_ROOT=%s\n' "$WORKTREE_ROOT"
          } >> "$GITHUB_ENV"
          if [ -n "$RESOLVED_BASE_SHA" ] && git worktree add --detach "$WORKTREE_ROOT/base" "$RESOLVED_BASE_SHA"; then
            {
              printf 'BASE_DIR=%s\n' "$WORKTREE_ROOT/base"
              printf 'RESOLVED_BASE_SHA=%s\n' "$RESOLVED_BASE_SHA"
            } >> "$GITHUB_ENV"
            echo "Using base commit $RESOLVED_BASE_SHA."
          else
            echo "BASE_BENCHMARKS_AVAILABLE=false" >> "$GITHUB_ENV"
          fi
      # Build both JMH jars before measuring either revision. Builds are CPU- and IO-heavy,
      # so building and measuring one revision at a time would bias results with different
      # thermal, cache, and CPU-frequency state on the shared runner.
      - name: "🔨 Build paired JMH jars"
        timeout-minutes: 60
        run: |
          base_build_ok=false
          head_build_ok=false
          if [ "${BASE_BENCHMARKS_AVAILABLE:-true}" = "true" ] && [ -f "$BASE_DIR/grails-benchmarks/build.gradle" ]; then
            if (
              cd "$BASE_DIR"
              ./gradlew :grails-benchmarks:jmhJar --max-workers=4
            ); then
              base_build_ok=true
              echo "Built base benchmark at $RESOLVED_BASE_SHA."
            else
              echo "BASE benchmark JAR build failed."
            fi
          else
            echo "BASE does not contain grails-benchmarks; comparison will use HEAD-only mode."
          fi
          if (
            cd "$HEAD_DIR"
            ./gradlew :grails-benchmarks:jmhJar --max-workers=4
          ); then
            head_build_ok=true
            echo "Built HEAD benchmark at $HEAD_SHA."
          else
            echo "HEAD benchmark JAR build failed."
          fi
          {
            printf 'BASE_BUILD_OK=%s\n' "$base_build_ok"
            printf 'HEAD_BUILD_OK=%s\n' "$head_build_ok"
          } >> "$GITHUB_ENV"
      - name: "🛑 Stop Gradle daemons before measurement"
        if: always()
        run: |
          ./gradlew --stop || true
          if [ -n "${BASE_DIR:-}" ] && [ -d "$BASE_DIR" ]; then (cd "$BASE_DIR" && ./gradlew --stop) || true; fi
          if [ -n "${HEAD_DIR:-}" ] && [ -d "$HEAD_DIR" ]; then (cd "$HEAD_DIR" && ./gradlew --stop) || true; fi
      # Two forks, three warmup iterations, and five measurement iterations balance PR latency
      # against confidence. Reversing shard order cancels first-versus-second runner-state bias.
      - name: "🌡️ Run paired JMH benchmarks"
        timeout-minutes: 60
        run: |
          # JMH writes results incrementally, so a run that dies partway can leave a file that
          # parses perfectly while describing only some of the benchmarks. The report job decides
          # completeness from which files exist, so a partial file would be indistinguishable from
          # a good one. Write to a staging path and publish it only on success, so a failed run
          # leaves NO file rather than a plausible one.
          run_benchmark() {
            local revision_dir="$1"
            local result_file="$2"
            local staging_file="$result_file.partial"
            rm -f "$staging_file" "$result_file"
            if (
              cd "$revision_dir" && ./gradlew --no-daemon :grails-benchmarks:jmh \
                -Pjmh.include="$JMH_INCLUDE" \
                -Pjmh.forks=2 \
                -Pjmh.warmupIterations=3 \
                -Pjmh.iterations=5 \
                -Pjmh.resultFile="$staging_file" \
                -Pjmh.profilers=gc \
                --max-workers=4
            ) && [ -s "$staging_file" ] && mv "$staging_file" "$result_file"; then
              return 0
            fi
            rm -f "$staging_file"
            return 1
          }

          base_run_failed=false
          head_run_failed=false
          run_base_benchmark() {
            if [ "${BASE_BUILD_OK:-false}" != "true" ]; then
              echo "BASE benchmark execution skipped because its JAR was not built."
            elif ! run_benchmark "$BASE_DIR" "$RESULT_DIR/base.json"; then
              base_run_failed=true
              echo "BASE benchmark execution failed."
            fi
          }
          run_head_benchmark() {
            if [ "${HEAD_BUILD_OK:-false}" != "true" ]; then
              echo "HEAD benchmark execution skipped because its JAR was not built."
            elif ! run_benchmark "$HEAD_DIR" "$RESULT_DIR/head.json"; then
              head_run_failed=true
              echo "HEAD benchmark execution failed."
            fi
          }

          if [ "$SHARD" = "a" ]; then
            run_base_benchmark
            run_head_benchmark
          else
            run_head_benchmark
            run_base_benchmark
          fi
          {
            printf 'BASE_RUN_FAILED=%s\n' "$base_run_failed"
            printf 'HEAD_RUN_FAILED=%s\n' "$head_run_failed"
          } >> "$GITHUB_ENV"
      # The comparison is advisory. The comparison tool exits successfully for regressions, and this
      # step is non-blocking even if results are incomplete because a benchmark execution failed.
      # No --pr-number is passed here on purpose: this job renders the report only. The separate
      # report job owns comment posting, so a two-shard matrix cannot produce duplicate comments.
      - name: "📊 Compare JMH results"
        if: always()
        continue-on-error: true
        run: |
          report_file="$REPORT_DIR/comparison.md"
          {
            printf '### JMH shard `%s`\n\n' "$SHARD"
            if [ "${BASE_RUN_FAILED:-false}" = "true" ]; then
              printf 'BASE benchmark execution failed.\n\n'
            fi
            if [ "${BASE_BUILD_OK:-false}" != "true" ]; then
              printf 'BASE benchmark JAR was not built.\n\n'
            fi
            if [ "${HEAD_RUN_FAILED:-false}" = "true" ]; then
              printf 'HEAD benchmark execution failed.\n\n'
            fi
            if [ "${HEAD_BUILD_OK:-false}" != "true" ]; then
              printf 'HEAD benchmark JAR was not built.\n\n'
            fi
          } > "$report_file"
          comparison_file="$RUNNER_TEMP/jmh-comparison-$SHARD.md"
          if [ ! -f "$RESULT_DIR/head.json" ]; then
            printf 'HEAD benchmark result was not produced.\n' >> "$report_file"
          elif [ "${BASE_BUILD_OK:-false}" = "true" ] && [ -f "$RESULT_DIR/base.json" ]; then
            ./gradlew -q --console=plain :grails-benchmarks:jmhCompare --args="--head $RESULT_DIR/head.json --base $RESULT_DIR/base.json --output $comparison_file"
            cat "$comparison_file" >> "$report_file"
          else
            ./gradlew -q --console=plain :grails-benchmarks:jmhCompare --args="--head $RESULT_DIR/head.json --output $comparison_file"
            cat "$comparison_file" >> "$report_file"
          fi
      - name: "📋 Publish JMH report in job summary"
        if: always()
        run: |
          if [ -f "$REPORT_DIR/comparison.md" ]; then
            cat "$REPORT_DIR/comparison.md" >> "$GITHUB_STEP_SUMMARY"
          else
            printf '## JMH benchmark comparison\n\nNo comparison report was produced.\n' >> "$GITHUB_STEP_SUMMARY"
          fi
      - name: "📤 Upload JMH artifacts"
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: jmh-results-${{ matrix.shard }}
          path: |
            jmh-results/${{ matrix.shard }}/
            jmh-reports/${{ matrix.shard }}/
          if-no-files-found: warn
      - name: "🧹 Remove paired worktrees"
        if: always()
        run: |
          WORKTREE_ROOT="$RUNNER_TEMP/jmh-worktrees/$SHARD"
          git worktree remove --force "$WORKTREE_ROOT/base" || true
          git worktree remove --force "$WORKTREE_ROOT/head" || true

  report:
    name: "Publish JMH benchmark comparison"
    needs: benchmark
    if: >-
      always() && github.event_name == 'pull_request' &&
      contains(github.event.pull_request.labels.*.name, 'performance') &&
      (github.event.action != 'labeled' || github.event.label.name == 'performance') &&
      github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      pull-requests: write
    env:
      PR_NUMBER: ${{ github.event.pull_request.number }}
      REPOSITORY: ${{ github.repository }}
      EXPECTED_SHARDS: "a b"
    steps:
      # This job runs only for same-repository pull requests, whose authors already hold push
      # access, so checking out the merge commit to run the Gradle-based comparison tool is not a privilege
      # escalation. Fork pull requests never reach this job: their pull_request token is
      # read-only and commenting would fail with 403, so they receive the per-shard job summary
      # and the uploaded artifacts instead.
      - name: "📥 Checkout repository"
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: "☕️ Setup JDK"
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
        with:
          distribution: liberica
          java-version: 21
      - name: "🐘 Setup Gradle"
        uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
        with:
          cache-provider: basic # 'basic' uses the MIT-licensed, open-source cache provider; the default 'enhanced' provider (v6+) is proprietary (Gradle commercial Terms of Use)
          develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      - name: "📥 Download JMH artifacts"
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: benchmark-artifacts
      # Both shards measured the same two revisions in opposite orders. Only complete same-shard
      # base/head pairs are pooled, preserving runner identity while letting the alternating order
      # cancel ordering bias. Incomplete shards are reported and never cross-paired.
      - name: "📊 Pool shard results and compare"
        id: compare
        run: |
          collected_base="$RUNNER_TEMP/collected-base"
          collected_head="$RUNNER_TEMP/collected-head"
          pooled_base="$RUNNER_TEMP/pooled-base"
          pooled_head="$RUNNER_TEMP/pooled-head"
          head_only="$RUNNER_TEMP/head-only"
          mkdir -p "$collected_base" "$collected_head" "$pooled_base" "$pooled_head" "$head_only"
          # Seeded from the matrix rather than from discovered files: a shard that failed before
          # uploading anything has no file to discover, and would otherwise be omitted from the
          # report entirely instead of being named as missing.
          declare -A observed_shards=()
          expected_shard_files=""
          for shard in $EXPECTED_SHARDS; do
            observed_shards["$shard"]=true
            expected_shard_files="${expected_shard_files:+$expected_shard_files,}shard-$shard.json"
          done
          while IFS= read -r -d '' result_file; do
            shard=""
            if [[ "$result_file" =~ /jmh-results-([^/]+)/ ]]; then
              shard="${BASH_REMATCH[1]}"
            elif [[ "$result_file" =~ /jmh-results/([^/]+)/ ]]; then
              shard="${BASH_REMATCH[1]}"
            else
              printf 'Unable to identify shard for result %s; ignoring it.\n' "$result_file"
              continue
            fi
            observed_shards["$shard"]=true
            case "$(basename "$result_file")" in
              base.json) cp "$result_file" "$collected_base/shard-$shard.json" ;;
              head.json) cp "$result_file" "$collected_head/shard-$shard.json" ;;
            esac
          done < <(find benchmark-artifacts -type f \( -name 'base.json' -o -name 'head.json' \) -print0)

          complete_shards=()
          dropped_shards=()
          if [ "${#observed_shards[@]}" -gt 0 ]; then
            for shard in "${!observed_shards[@]}"; do
              if [ -f "$collected_base/shard-$shard.json" ] && [ -f "$collected_head/shard-$shard.json" ]; then
                cp "$collected_base/shard-$shard.json" "$pooled_base/shard-$shard.json"
                cp "$collected_head/shard-$shard.json" "$pooled_head/shard-$shard.json"
                complete_shards+=("$shard")
              else
                dropped_shards+=("$shard")
              fi
              if [ -f "$collected_head/shard-$shard.json" ]; then
                cp "$collected_head/shard-$shard.json" "$head_only/shard-$shard.json"
              fi
            done
          fi

          report_file="$RUNNER_TEMP/pooled-comparison.md"
          comparison_file="$RUNNER_TEMP/jmh-comparison.md"
          {
            printf '### JMH Shard Pairing\n\n'
            printf '**Complete shard pairs used:** %s\n' "${#complete_shards[@]}"
            if [ "${#dropped_shards[@]}" -gt 0 ]; then
              printf '**Dropped incomplete shards:**'
              printf ' `%s`' "${dropped_shards[@]}"
              printf '\n'
            fi
            printf '\n'
          } > "$report_file"
          should_post=false
          if [ "${#complete_shards[@]}" -gt 0 ]; then
            should_post=true
            ./gradlew -q --console=plain :grails-benchmarks:jmhCompare --args="--head $pooled_head --base $pooled_base --expected-shards $expected_shard_files --output $comparison_file"
            cat "$comparison_file" >> "$report_file"
          elif compgen -G "$head_only/*.json" > /dev/null; then
            should_post=true
            printf 'No complete same-runner base and HEAD shard pair was available. Reporting available HEAD results only.\n\n' >> "$report_file"
            ./gradlew -q --console=plain :grails-benchmarks:jmhCompare --args="--head $head_only --output $comparison_file"
            cat "$comparison_file" >> "$report_file"
          else
            printf '### JMH Benchmark Report\n\nNo HEAD benchmark results were produced for this run. The previous JMH report comment on this PR (if any) was left unchanged.\n' >> "$report_file"
          fi
          cat "$report_file" >> "$GITHUB_STEP_SUMMARY"
          printf 'report_file=%s\n' "$report_file" >> "$GITHUB_OUTPUT"
          printf 'should_post=%s\n' "$should_post" >> "$GITHUB_OUTPUT"
      - name: "💬 Post or update JMH PR comment"
        if: steps.compare.outputs.should_post == 'true'
        continue-on-error: true
        env:
          REPORT_FILE: ${{ steps.compare.outputs.report_file }}
          GITHUB_TOKEN: ${{ github.token }}
        run: ./gradlew -q --console=plain :grails-benchmarks:jmhCompare --args="--post-file $REPORT_FILE --repo $REPOSITORY --pr-number $PR_NUMBER"