test.yml

name: Test benchmarks
on:
  push:
    branches:
      - main
  pull_request:
  schedule:
    - cron: "00 15 * * *"
env:
  BUNDLE_VERSION: system
jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        ruby: [ruby, head, truffleruby]
    if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
    steps:
      - uses: actions/checkout@v3
      - name: Cache apt packages
        uses: actions/cache@v4
        with:
          path: |
            /var/cache/apt/archives
            /var/lib/apt/lists
          key: ${{ runner.os }}-apt-imagemagick-${{ hashFiles('.github/workflows/test.yml') }}
          restore-keys: |
            ${{ runner.os }}-apt-imagemagick-
      - name: Install ImageMagick dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends libmagickwand-dev
      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: ${{ matrix.ruby }}
          bundler: ${{ matrix.ruby == 'head' && 'default' || 'Gemfile.lock' }}
          bundler-cache: true

      - name: Run tests
        run: bundle exec rake test

  benchmark-default:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - ruby: ruby
          - ruby: head
          - ruby: truffleruby
            # knucleotide: needs fork
            # lobsters: commonmarker fails to install
            # ruby-lsp: rbs_extension.so: undefined symbol: ruby_vm_at_exit
            # shipit: cannot load such file -- pty
            extra_args: "--excludes=knucleotide,lobsters,ruby-lsp,shipit"
    if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: ${{ matrix.ruby }}

      - name: Test run_benchmarks.rb
        run: ./run_benchmarks.rb ${{ matrix.extra_args }}
        env:
          WARMUP_ITRS: '1'
          MIN_BENCH_ITRS: '1'
          MIN_BENCH_TIME: '0'

  benchmark-ractor:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        ruby: [head]
    if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: ${{ matrix.ruby }}

      - name: Test run_benchmarks.rb - Ractors
        run: ./run_benchmarks.rb --category=ractor
        env:
          WARMUP_ITRS: '1'
          MIN_BENCH_ITRS: '1'
          MIN_BENCH_TIME: '0'

  benchmark-ractor-only:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        ruby: [head]
    if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: ${{ matrix.ruby }}

      - name: Test run_benchmarks.rb - Ractor Only
        run: ./run_benchmarks.rb --category=ractor-only
        env:
          WARMUP_ITRS: '1'
          MIN_BENCH_ITRS: '1'
          MIN_BENCH_TIME: '0'

  benchmark-graph:
    runs-on: ubuntu-latest
    if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
    steps:
      - uses: actions/checkout@v3
      - name: Cache apt packages
        uses: actions/cache@v4
        with:
          path: |
            /var/cache/apt/archives
            /var/lib/apt/lists
          key: ${{ runner.os }}-apt-imagemagick-${{ hashFiles('.github/workflows/test.yml') }}
          restore-keys: |
            ${{ runner.os }}-apt-imagemagick-
      - name: Install ImageMagick dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends libmagickwand-dev
      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: ruby

      - name: Test run_benchmarks.rb --graph
        run: ./run_benchmarks.rb --graph fib
        env:
          WARMUP_ITRS: '1'
          MIN_BENCH_ITRS: '1'
          MIN_BENCH_TIME: '0'

  benchmark-run-once:
    runs-on: ubuntu-latest
    if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: head

      - name: Run a benchmark once directly with Ruby
        run: ruby --yjit-stats benchmarks/railsbench/benchmark.rb
        env:
          WARMUP_ITRS: '0'
          MIN_BENCH_ITRS: '1'
          MIN_BENCH_TIME: '0'

  test-windows:
    runs-on: windows-latest
    env:
      # rmagick does not build against MSYS2's ImageMagick
      BUNDLE_WITHOUT: graph
    strategy:
      fail-fast: false
      matrix:
        ruby: [ruby, head]
    if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: ${{ matrix.ruby }}
          bundler: ${{ matrix.ruby == 'head' && 'default' || 'Gemfile.lock' }}
          bundler-cache: true

      - name: Run tests
        run: bundle exec rake test

  benchmark-default-windows:
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        ruby: [ruby, head]
    env:
      # erubi-rails, lobsters, railsbench: nokogiri 1.18.9 builds from source and its libiconv fails to compile
      # graphql-native: graphql-c_parser's FLOAT, INT and INPUT tokens clash with Windows headers
      # lobsters: stackprof fails to build
      EXCLUDES: erubi-rails,graphql-native,lobsters,railsbench
    if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: ${{ matrix.ruby }}

      - name: Test run_benchmarks.rb
        run: ruby run_benchmarks.rb --excludes=${{ env.EXCLUDES }}
        env:
          WARMUP_ITRS: '1'
          MIN_BENCH_ITRS: '1'
          MIN_BENCH_TIME: '0'

  all-tests:
    runs-on: ubuntu-latest
    needs: [test, test-windows, benchmark-default, benchmark-default-windows, benchmark-ractor, benchmark-ractor-only, benchmark-graph, benchmark-run-once]
    if: always()
    steps:
      - name: Check all job results
        run: |
          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
            echo "One or more jobs failed"
            exit 1
          elif [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
            echo "One or more jobs were cancelled"
            exit 1
          else
            echo "All jobs completed successfully"
          fi