benchmark.yml

on:
  workflow_call:
    inputs:
      setup:
        type: string
        required: true
  workflow_dispatch:
    inputs:
      setup:
        type: choice
        options:
          - benchmarks-all
          - benchmarks-default
          - bm-basics-fp32-single
          - bm-basics-fp32-multi
          - bm-basics-fp64-single
          - bm-basics-fp64-multi
          - bm-basics-bf16-single
          - bm-basics-bf16-multi
          - bm-basics-fp16-single
          - bm-basics-fp16-multi
          - bm-batch-iter-fp32-single
          - bm-batch-iter-fp32-multi
          - bm-batch-iter-fp64-single
          - bm-batch-iter-fp64-multi
          - bm-batch-iter-bf16-single
          - bm-batch-iter-bf16-multi
          - bm-batch-iter-fp16-single
          - bm-batch-iter-fp16-multi
          - bm-updated-fp32-single
          - bm-spaces
        description: 'Benchmarks set to run'
        default: benchmarks-all

jobs:
  start-runner:
    name: Start self-hosted EC2 runner
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}
      - name: Start EC2 runner
        id: start-ec2-runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: start
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          # Ubuntu 22.04 128GB Storage AMI
          ec2-image-id: ami-0ba430d4b7b64de57
          ec2-instance-type: r7i.xlarge
          subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
          security-group-id: ${{ secrets.AWS_EC2_SG_ID }}

  benchmark:
    name: Run the benchmarks on the runner
    needs: start-runner # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.number && format('refs/pull/{0}/merge', github.event.number) || github.head_ref }}
      - name: Print runner info
        run: |
          printf "Runner lscpu:\n$(lscpu)\n"
          printf "Runner lsmem:\n$(lsmem)\n"
          printf "Runner nproc:\n$(nproc)\n"
          printf "Runner uname:\n$(uname -a)\n"

      - name: Install benchmark dependencies
        run: |
             sudo .install/install_script.sh
             sudo apt install python3-pip -y
             pip3 install --upgrade pip PyYAML setuptools redisbench-admin
             pip3 install -r requirements.txt

      # - name: stress test
      #   run: |
      #     sudo apt install stress-ng -qqy
      #     uptime
      #     stress-ng -c 1 --timeout 60s --metrics-brief
      #     uptime
      #     stress-ng --stream 1 -t 60 --metrics-brief
      #     uptime
      #     stress-ng --ipsec-mb=1 -t 60 --metrics-brief
      #     uptime

      # TODO: remove "--no-check-certificate" when possible
      - name: Download pre-generated indices
        timeout-minutes: 20
        run: ./tests/benchmark/bm_files.sh ${{ inputs.setup }}
      - name: Benchmark
        timeout-minutes: 120
        run: make benchmark BM_FILTER=${{ inputs.setup }}

      - name: Collect results
        run: |
          ./tests/benchmark/benchmarks.sh ${{ inputs.setup }} | xargs -P 0 -I {} redisbench-admin export     \
          --redistimeseries_host      ${{ secrets.PERFORMANCE_RTS_HOST }}           \
          --redistimeseries_port      ${{ secrets.PERFORMANCE_RTS_PORT }}           \
          --redistimeseries_user      default                                       \
          --redistimeseries_pass      '${{ secrets.PERFORMANCE_RTS_AUTH }}'         \
          --github_repo               ${{ github.event.repository.name }}           \
          --github_org                ${{ github.repository_owner }}                \
          --github_branch             ${{ github.head_ref || github.ref_name }}     \
          --github_actor              ${{ github.triggering_actor }}                \
          --results-format            google.benchmark                              \
          --benchmark-result-file     {}_results.json

  stop-runner:
    name: Stop self-hosted EC2 runner
    needs:
      - start-runner # required to get output from the start-runner job
      - benchmark # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}