model-inference-with-timing.yml

name: Inference with metrics and timing evaluation

on:
  push:
    paths:
      - "**/**/*.onnx"
  workflow_dispatch:

permissions:
  contents: write
  pull-requests: write

jobs:
  select-runner:
    runs-on: ubuntu-22.04
    outputs:
      runner: ${{ steps.select_runner.outputs.runner }}
    steps:
      - uses: actions/checkout@v4
      - name: Select runner
        id: select_runner
        run: |
          if [ "${{ github.repository }}" = "NeuroDesk/cvpr-sam-on-laptop-2024" ]; then
            echo "runner=self-hosted" >> $GITHUB_OUTPUT
          else
            echo "runner=ubuntu-22.04" >> $GITHUB_OUTPUT
          fi

  list_models:
    needs: [select-runner]
    runs-on: ${{ needs.select-runner.outputs.runner }}
    outputs:
      model_list: ${{ steps.set_model_list.outputs.model_list }}
      all_models: ${{ steps.set_model_list.outputs.all_models }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Set up environment and data from osf
        id: set_model_list
        run: |
          python -m pip install --upgrade pip
          python -m pip install osfclient

          model_dir=$( osf --project u8tny list)
          all_models=$( echo "$model_dir" | awk -F '/' '/osfstorage\/checkpoints/ {print $3}' | sort -u | tr '\n' ' ')
          echo "all_models=${all_models}"
          model_list='['
          for MODEL in $(echo "${all_models}"); do
            echo "MODEL=${MODEL}"
            model_list+="\"${MODEL}\","
          done
          model_list=$(sed '$s/,$//' <<< $model_list)
          model_list+=']'
          echo "model_list=${model_list}"
          echo "model_list=${model_list}" >> $GITHUB_OUTPUT
          echo "all_models=${all_models}" >> $GITHUB_OUTPUT

  infer_new_model:
    needs: [select-runner, list_models]
    if: ${{ needs.list_models.outputs.model_list != '[]' }}
    runs-on: ${{ needs.select-runner.outputs.runner }}
    strategy:
      fail-fast: false
      matrix:
        models: ${{ fromJson(needs.list_models.outputs.model_list) }}
    steps:
      - name: Cleanup
        shell: bash
        run: |
          pwd
          ls -l
          echo ${{ secrets.USERPWD }} | sudo -S rm -rf test_demo
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Set up environment and data from osf for ${{ matrix.models }}
        id: set_model_list
        run: |
          python -m pip install --upgrade pip
          python -m pip install osfclient numpy nibabel pandas tqdm matplotlib scipy
          osf --project u8tny fetch test_demo.tar.xz 
          osf --project u8tny fetch checkpoints/${{ matrix.models }}/${{ matrix.models }}_encoder.onnx
          osf --project u8tny fetch checkpoints/${{ matrix.models }}/${{ matrix.models }}_decoder.onnx
          tar -xvf test_demo.tar.xz
          rm test_demo.tar.xz
          mv ${{ matrix.models }}_encoder.onnx ${{ matrix.models }}_decoder.onnx work_dir/
          mv imgs gts test_demo
      - name: Run inference on ${{ matrix.models }}
        shell: bash
        run: |
          mkdir segs && chmod -R 777 ./*
          export DOCKER_IMAGE_NAME=hawken50
          echo ${{ secrets.USERPWD }} | sudo -S docker build -f Dockerfile -t "$DOCKER_IMAGE_NAME" .
          echo ${{ secrets.USERPWD }} | sudo -S docker container run -m 8G --name "$DOCKER_IMAGE_NAME" --rm "$DOCKER_IMAGE_NAME":latest /bin/bash -c "ls"
          echo ${{ secrets.USERPWD }} | sudo -S docker container run -m 8G --name "$DOCKER_IMAGE_NAME" --rm "$DOCKER_IMAGE_NAME":latest /bin/bash -c "ls work_dir"
          echo ${{ secrets.USERPWD }} | ./evaluate.sh .

      - name: Display accuracy, efficiency, and running time
        run: |
          printf "Metrics:\n" >> $GITHUB_STEP_SUMMARY
          cat ./metrics.csv >> $GITHUB_STEP_SUMMARY
          printf "\n\n" >> $GITHUB_STEP_SUMMARY
          printf "Running time:\n" >> $GITHUB_STEP_SUMMARY
          cat ./running_time.csv >> $GITHUB_STEP_SUMMARY
          mkdir ${{ matrix.models }}_results
          mv metrics.csv running_time.csv ${{ matrix.models }}_results/

      - name: Archive metrics and running time
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.models }}_results
          path: |
            ${{ matrix.models }}_results

  plot_results:
    needs: [list_models, infer_new_model]
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Set up environment and data from osf
        run: |
          python -m pip install --upgrade pip
          python -m pip install numpy tqdm matplotlib scipy pandas

      - name: Download result artifacts
        uses: actions/download-artifact@v4

      - name: Plot results
        run: |
          python plot_results.py -models ${{ needs.list_models.outputs.all_models }}
          ls
          echo "![Image](./pareto_front.png)" >> "$GITHUB_STEP_SUMMARY"