LAB-1491 ray jobs (#992) #1776
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # This workflow will build a golang project and runs CI tests | |
| name: CI | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| preview-enabled: | |
| description: 'check whether to enable preview by setting up tunnel and keep environment up. Default `true`' | |
| required: false | |
| default: true | |
| type: choice | |
| options: | |
| - true | |
| - false | |
| preview-timeout: | |
| description: 'preview environment timeout. This setting is how long the environment would be available for. Set it less than Github Action timeout of 360 minutes. This value is passed into `sleep` command, so follow the sleep command syntax. Default `360m`' | |
| required: false | |
| type: string | |
| default: "360m" | |
| push: | |
| branches: | |
| - "main" | |
| - "staging" | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'infrastructure/**' | |
| pull_request: | |
| branches: | |
| - "main" | |
| - "staging" | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'infrastructure/**' | |
| # Only run one at a time | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Download Bacalhau CLI for troubleshooting | |
| ci-setup-bacalhau: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| # - macos-13 | |
| bacalhau_version: | |
| - 1.2.0 | |
| runs-on: ${{ matrix.os }} | |
| environment: ci | |
| steps: | |
| - name: Download bacalhau | |
| run: | | |
| # Download bacalhau plugin | |
| if [[ "$OSTYPE" == "linux-gnu"* ]]; then | |
| curl -sSL https://github.com/bacalhau-project/bacalhau/releases/download/v${{ matrix.bacalhau_version }}/bacalhau_v${{ matrix.bacalhau_version }}_linux_amd64.tar.gz -o bacalhau.tgz | |
| elif [[ "$OSTYPE" == "darwin"* ]]; then | |
| curl -sSL https://github.com/bacalhau-project/bacalhau/releases/download/v${{ matrix.bacalhau_version }}/bacalhau_v${{ matrix.bacalhau_version }}_darwin_arm64.tar.gz -o bacalhau.tgz | |
| fi | |
| tar -zxvf bacalhau.tgz | |
| - name: upload bacalhau plugin to be used later | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: bacalhau-binary-${{ matrix.os }}-${{ matrix.bacalhau_version }} | |
| path: ./bacalhau | |
| ci: | |
| needs: | |
| # - ci-setup-compose | |
| - ci-setup-bacalhau | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| # - macos-13 | |
| bacalhau_version: | |
| - 1.2.0 | |
| network_mode: | |
| - public | |
| - private | |
| runs-on: ${{ matrix.os }} | |
| environment: ci | |
| env: | |
| BACALHAU_VERSION: ${{ matrix.bacalhau_version }} | |
| # Setting it at workflow level to be used by all the steps | |
| BACALHAU_API_HOST: "127.0.0.1" | |
| LOG_LEVEL: trace | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: 1.20.3 | |
| cache-dependency-path: go.sum | |
| - name: Install dependencies | |
| run: go mod download | |
| - name: Build | |
| run: go build | |
| - name: Upload plex binary | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: plex-binary-${{ matrix.network_mode }}-${{ matrix.os }}-${{ matrix.bacalhau_version }} | |
| path: ./plex | |
| - name: download bacalhau binary | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: bacalhau-binary-${{ matrix.os }}-${{ matrix.bacalhau_version }} | |
| - name: Setup docker (missing on MacOS) | |
| if: runner.os == 'macos' | |
| run: | | |
| brew install colima docker docker-compose | |
| colima start | |
| # For testcontainers to find the Colima socket | |
| # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running | |
| sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock | |
| mkdir -p ~/.docker/cli-plugins | |
| ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose | |
| docker version | |
| docker compose version | |
| - name: Setup docker compose plugin | |
| if: runner.os != 'macos' | |
| run: | | |
| # Output version info | |
| docker version | |
| docker compose version | |
| - name: Bring tunnel for previewing environment | |
| if: inputs.preview-enabled == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| # | |
| set -x | |
| # Create empty file to avoid issues | |
| touch ./docker/.env.tunnel | |
| # Setup docker compose tunnel | |
| docker compose -f docker-compose.tunnel.yml up -d --wait | |
| # copy tunnel info to .env | |
| cat docker/.env.tunnel >> .env | |
| - name: docker compose build | |
| run: | | |
| # Build compose | |
| docker compose build --parallel | |
| - name: Bring up the public stack | |
| if: matrix.network_mode == 'public' | |
| env: | |
| STRIPE_WEBHOOK_SECRET_KEY: ${{ secrets.STRIPE_WEBHOOK_SECRET_KEY }} | |
| run: | | |
| # Compose up | |
| docker compose -f docker-compose.yml up -d --wait | |
| - name: Bring up the private stack | |
| if: matrix.network_mode == 'private' | |
| env: | |
| STRIPE_WEBHOOK_SECRET_KEY: ${{ secrets.STRIPE_WEBHOOK_SECRET_KEY }} | |
| run: | | |
| # Compose up | |
| docker compose -f docker-compose.yml -f docker-compose.private.yml up -d --wait | |
| - name: Run docker compose ps | |
| run: | | |
| # Inspect number of running containers | |
| docker compose ps | |
| - name: fix permissions | |
| if: always() | |
| run: | | |
| set -x | |
| # Execute permission | |
| chmod +x ./bacalhau ./plex | |
| - name: Go Test | |
| run: go test ./... -v | |
| - name: Run Gateway Integration Tests | |
| uses: ./.github/actions/gateway-tests | |
| - name: Test log streaming | |
| # run always even when | |
| if: always() | |
| run: | | |
| ./bacalhau docker run -f ubuntu -- /bin/bash -c 'for i in {1..5}; do echo Hello World; sleep 1; done' | tee /tmp/output.log | |
| cat /tmp/output.log | |
| - name: Run docker compose logs | |
| # run always even when | |
| if: always() | |
| run: | | |
| docker compose logs | |
| - name: Run docker logs to get additional logs | |
| # run always even when | |
| if: always() | |
| run: | | |
| docker ps -a | |
| for container in $(docker ps -qa); do echo ${container};docker logs ${container};done | |
| - name: upload outputs | |
| # run always even when | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ci-${{ matrix.network_mode }}-output-${{ matrix.os }}-${{ matrix.bacalhau_version }} | |
| path: | | |
| plex_run_output.log | |
| job-* | |
| jobs/ | |
| - name: Tunnel URLs | |
| if: inputs.preview-enabled == 'true' | |
| run: | | |
| cat ./docker/.env.tunnel | |
| - name: Post tunnel URLs to PR | |
| if: inputs.preview-enabled == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| # | |
| set -x | |
| # Comment output | |
| source ./docker/.env.tunnel | |
| # Adding comment | |
| echo '---' >> comment.md | |
| echo >> comment.md | |
| echo ':information_source: NOTE' >> comment.md | |
| echo >> comment.md | |
| echo 'The URLs will only be available for 6 hours' >> comment.md | |
| echo '---' >> comment.md | |
| echo '## Preview URLs' >> comment.md | |
| echo 'Bacalhau Version: ${{ matrix.bacalhau_version }}' >> comment.md | |
| echo 'OS: ${{ matrix.os }}' >> comment.md | |
| echo 'Network Mode: ${{ matrix.network_mode }}' >> comment.md | |
| echo "| Endpoint | URL |" >> comment.md | |
| echo "|---|---|" >> comment.md | |
| echo "| FRONTEND_URL | ${FRONTEND_URL} |" >> comment.md | |
| echo "| BACKEND_URL | ${NEXT_PUBLIC_BACKEND_URL} |" >> comment.md | |
| echo "| IPFS_GATEWAY_URL | ${NEXT_PUBLIC_IPFS_GATEWAY_ENDPOINT} |" >> comment.md | |
| # Comment on PR with URLs | |
| PR_NUMBER=$(gh pr view --json number | jq -r '.number') | |
| if [[ ! -z "${PR_NUMBER}" ]]; then | |
| gh pr comment "${PR_NUMBER}" --body-file comment.md | |
| fi | |
| - name: Sleep for preview-timeout | |
| if: inputs.preview-enabled == 'true' | |
| run: sleep ${{ inputs.preview-timeout }} |