Smoke test #243
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Smoke test | |
| on: | |
| # regular runs | |
| schedule: | |
| - cron: '0 0,6,12,18 * * *' | |
| # allow button click | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| environment: Scheduled Testing | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Store commit | |
| run: | | |
| echo "GIT_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - name: cache poetry install | |
| uses: actions/cache@v3 | |
| id: cache-poetry | |
| with: | |
| path: ~/.local | |
| key: poetry-1.7.1-0 | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: 1.7.1 | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| - name: cache deps | |
| id: cache-deps | |
| uses: actions/cache@v3 | |
| with: | |
| path: .venv | |
| key: pydeps-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies with caching | |
| run: poetry install --no-interaction --no-root | |
| if: steps.cache-deps.outputs.cache-hit != 'true' | |
| - name: Install with plugins | |
| run: poetry install --no-interaction --sync --extras all_plugins | |
| - name: Write secrets | |
| env: | |
| SECRETS_CONFIG: | | |
| [anthropic] | |
| api_key = "${{ secrets.ANTHROPIC_API_KEY }}" | |
| [together] | |
| api_key = "${{ secrets.TOGETHER_API_KEY }}" | |
| [openai] | |
| api_key = "${{ secrets.OPENAI_API_KEY }}" | |
| [hugging_face] | |
| token = "${{ secrets.HUGGING_FACE_TOKEN }}" | |
| [google_ai] | |
| api_key = "${{ secrets.GOOGLE_AI_API_KEY }}" | |
| [demo] | |
| api_key="12345" | |
| run: | | |
| mkdir -p config | |
| echo "$SECRETS_CONFIG" > config/secrets.toml | |
| - name: Test object creation with plugins | |
| run: | | |
| source .venv/bin/activate | |
| pytest --expensive-tests | |
| - name: Test v0.5 run | |
| run: | | |
| source .venv/bin/activate | |
| modelbench benchmark -m 1 -v 0.5 | |
| - name: Test v1.0 run | |
| run: | | |
| source .venv/bin/activate | |
| modelbench benchmark -m 1 -v 1.0 | |
| - name: Ensure the artifact published on Pypi still works as expected | |
| run: | | |
| rm -rf .venv | |
| mkdir -p ../installation/config | |
| cat ./tests/modelgauge_tests/data/install_pyproject.toml > ../installation/pyproject.toml | |
| cd ../installation | |
| touch ./config/secrets.toml | |
| poetry lock | |
| poetry install --no-root | |
| poetry run modelgauge list-tests | |
| - name: Discord notification | |
| if: failure() | |
| env: | |
| DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} | |
| uses: Ilshidur/[email protected] | |
| with: | |
| args: 'The smoke test for {{ EVENT_PAYLOAD.repository.full_name }} has failed. See {{ EVENT_PAYLOAD.repository.html_url }}/actions' |