Using package-benchmark for benchmarking Swift code

benchmark.yml

name: benchmark

on:
  workflow_dispatch:
  pull_request: { types: [opened, reopened, synchronize] }
  push: { branches: [main] }

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  benchmark-branch:
    # Using GitHub Actions runners is not recommended for non-allocation benchmarks. They're not consistent enough.
    runs-on: ubuntu-latest

    timeout-minutes: 60

    container: swift:5.10-noble

    env:
      PR_COMMENT: null # will be populated later

    steps:
      - name: Configure git
        shell: bash
        run: |
          set -eu

          # For packages with private dependencies, you'll need to provide a custom token:
          # git config --global --add url."https://${{ secrets.GH_PAT }}@github.com/".insteadOf "https://github.com/"
          # git config --global --add url."https://${{ secrets.GH_PAT }}@github.com/".insteadOf "[email protected]:"
          git config --global --add safe.directory '*'

      - name: Check out ${{ github.event.repository.name }}
        uses: actions/checkout@v4

      # jemalloc is a dependency of the Benchmarking package
      - name: Install jemalloc
        run: |
          set -eu

          apt-get update -y
          apt-get install -y libjemalloc-dev

      - name: Restore .build
        id: "restore-cache"
        uses: runs-on/cache/restore@v4
        with:
          path: .build
          # e.g. 'my-repo-benchmark-build-Linux-c7008df8062ac4d5887ead9e59aa05829e'
          key: "${{ github.event.repository.name }}-benchmark-build-${{ runner.os }}-${{ github.event.pull_request.base.sha || github.event.after }}"
          restore-keys: "${{ github.event.repository.name }}-benchmark-build-${{ runner.os }}-"

      - name: Run Benchmarks Of Current Branch '${{ github.head_ref || github.ref_name }}'
        run: |
          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update "${{ github.head_ref || github.ref_name }}" --no-progress --quiet

      - name: Cache .build
        if: steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'workflow_dispatch'
        uses: runs-on/cache/save@v4
        with:
          path: .build
          key: "${{ github.event.repository.name }}-benchmark-build-${{ runner.os }}-${{ github.event.pull_request.base.sha || github.event.after }}"

      - name: Read Benchmark Result
        shell: bash
        id: read-benchmark
        run: |
          set -eu

          swift package benchmark baseline read \
            "${{ github.head_ref || github.ref_name }}" \
            --no-progress \
            --format markdown \
            >> result.text

          # Read the result to the output of the step
          echo "result<<EOF" >> $GITHUB_OUTPUT
          cat result.text >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - name: Compare Current Branch '${{ github.head_ref || github.ref_name }}' Against Thresholds
        id: compare-benchmark
        shell: bash
        run: |
          set -eu

          TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
          ENCODED_TIMESTAMP=$(date -u +"%Y-%m-%dT%H%%3A%M%%3A%SZ")
          TIMESTAMP_LINK="https://www.timeanddate.com/worldclock/fixedtime.html?iso=$ENCODED_TIMESTAMP"
          echo "## Benchmark check running at [$TIMESTAMP]($TIME_LINK)" >> summary.text

          # Disable 'set -e' to prevent the script from exiting on non-zero exit codes
          set +e
          swift package benchmark baseline check \
            --check-absolute-path $PWD/Benchmarks/Thresholds/ \
            --no-progress \
            --format markdown \
            >> summary.text
          echo "exit-status=$?" >> $GITHUB_OUTPUT
          set -e

          echo "summary<<EOF" >> $GITHUB_OUTPUT
          cat summary.text >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - name: Install curl and jq
        shell: bash
        run: |
          set -eu

          apt-get update -y
          apt-get install -y curl jq

      - name: Construct Comment
        shell: bash
        run: |
          set -eu

          echo 'PR_COMMENT<<EOF' >> $GITHUB_ENV

          # The fact that the comment starts with '## [Benchmark](' is used
          # in a later step to find this comment again.
          # Be wary of that when changing this line.
          echo "## [Benchmark](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) Report" >> $GITHUB_ENV

          EXIT_CODE="${{ steps.compare-benchmark.outputs.exit-status }}"
          case "$EXIT_CODE" in
              0)
                  echo "**✅ Pull request has no significant performance differences ✅**" >> $GITHUB_ENV
                  ;;
              1)
                  echo "**❌ Pull request has significant performance differences 📊**" >> $GITHUB_ENV
                  ;;
              2)
                  echo "**❌ Pull request has significant performance regressions 📉**" >> $GITHUB_ENV
                  ;;
              4)
                  echo "**❌ Pull request has significant performance improvements 📈**" >> $GITHUB_ENV
                  ;;
              *)
                  echo "**❌ Benchmark comparison failed to complete properly with exit code $EXIT_CODE ❌**" >> $GITHUB_ENV
                  ;;
          esac

          echo "<details>" >> $GITHUB_ENV
          echo "  <summary> Click to expand comparison result </summary>" >> $GITHUB_ENV
          echo "" >> $GITHUB_ENV
          echo '${{ steps.compare-benchmark.outputs.summary }}' >> $GITHUB_ENV
          echo "" >> $GITHUB_ENV
          echo "</details>" >> $GITHUB_ENV

          echo "" >> $GITHUB_ENV

          echo "<details>" >> $GITHUB_ENV
          echo "  <summary> Click to expand benchmark result </summary>" >> $GITHUB_ENV
          echo "" >> $GITHUB_ENV
          echo '${{ steps.read-benchmark.outputs.result }}' >> $GITHUB_ENV
          echo "" >> $GITHUB_ENV
          echo "</details>" >> $GITHUB_ENV

          echo 'EOF' >> $GITHUB_ENV

      - name: Output The Comment as Job Summary
        shell: bash
        run: |
          set -eu

          echo '${{ env.PR_COMMENT }}' >> $GITHUB_STEP_SUMMARY

      - name: Find Existing Comment ID
        if: github.event_name == 'pull_request'
        id: existing-comment
        shell: bash
        run: |
          set -eu

          # Known limitation: This only fetches the first 100 comments. This should not
          # matter much because a benchmark comment should have been sent early in the PR.
          curl -sL \
            -X GET \
            -H "Accept: application/vnd.github+json" \
            -H "Authorization: BEARER ${{ secrets.GITHUB_TOKEN }}" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            -o result.json \
            "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.number }}/comments?per_page=100"

          # Select the last comment that is done via the GitHub-Actions bot, and
          # has a body that starts with '## [Benchmark]('.
          # If available, we can just update this comment instead of creating new ones.
          EXISTING_COMMENT_ID=$(
            cat result.json |
              jq '
                [
                  .[] |
                    select(
                      .performed_via_github_app.owner.login == "github"
                        and
                      (.body | startswith("## [Benchmark]("))
                    )
                ][-1].id
              '
          )

          echo "existing-comment-id=$EXISTING_COMMENT_ID" >> $GITHUB_OUTPUT

      - name: Comment in PR
        if: github.event_name == 'pull_request'
        shell: bash
        run: |
          set -eu

          EXISTING_COMMENT_ID="${{ steps.existing-comment.outputs.existing-comment-id }}"
          # Need to provide the argument in a way that anything in the comment itself is not evaluated.
          BODY_JSON="$(jq -n -c --arg COMMENT "$(echo '${{ env.PR_COMMENT }}')" '{"body": $COMMENT}')"

          if [ "$EXISTING_COMMENT_ID" != "null" ]; then
            echo "Will update a comment: $EXISTING_COMMENT_ID"
            ENDPOINT="https://api.github.com/repos/${{ github.repository }}/issues/comments/$EXISTING_COMMENT_ID"
          else
            echo "Will create a new comment"
            ENDPOINT="https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.number }}/comments"
          fi

          curl -sL \
            -X POST \
            -H "Accept: application/vnd.github+json" \
            -H "Authorization: BEARER ${{ secrets.GITHUB_TOKEN }}" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            -d "$BODY_JSON" \
            "$ENDPOINT"

      - name: Exit With Correct Status
        run: |
          exit ${{ steps.compare-benchmark.outputs.exit-status }}