Skip to content

Smoke Tests

Smoke Tests #7120

Workflow file for this run

# Tests a broad set of Quarto functionality that users are likely to encounter.
# A failure indicates some signficant portion of functionality is likely to be broken.
name: Smoke Tests
on:
workflow_call:
inputs:
buckets:
description: "JSON string for buckets of tests to run in loop. Array of grouped tests."
required: true
type: string
time-test:
description: "Should we run tests to produce test file"
required: false
type: boolean
default: false
extra-r-packages:
description: "extra R package to install for the runs (like a dev version of one of the deps) - comma separated, passed to renv::install"
required: false
type: string
default: ""
workflow_dispatch:
inputs:
buckets:
description: "JSON string for buckets of tests to run in loop. Array of grouped tests."
required: false
type: string
default: ""
time-test:
description: "Should we run tests to produce test file"
required: false
type: boolean
default: false
extra-r-packages:
description: "extra R package to install for the runs (like a dev version of one of the deps) - comma separated, passed to renv::install"
required: false
type: string
default: ""
schedule:
- cron: "0 6 * * *"
jobs:
run-smokes:
name: Run smoke (${{ matrix.os }})${{ matrix.time-test && ' with timed file' || ''}}${{ inputs.extra-r-packages && ' - with some extra R packages'|| ''}}
if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'quarto-dev/quarto-cli')
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
time-test:
- ${{ inputs.time-test }}
exclude:
# only run timed tests on Linux
- os: windows-latest
time-test: true
runs-on: ${{ matrix.os }}
steps:
- name: Checkout Repo
uses: actions/checkout@v4
- name: Fix temp dir to use runner one (windows)
if: runner.os == 'Windows'
run: |
echo "TMPDIR=${{ runner.temp }}" >> $GITHUB_ENV
echo "TMP=${{ runner.temp }}" >> $GITHUB_ENV
echo "TEMP=${{ runner.temp }}" >> $GITHUB_ENV
shell: bash
- name: Set up R
uses: r-lib/actions/setup-r@v2
with:
r-version: "4.4.2"
use-public-rspm: true
# required to avoid rtools bin in path
windows-path-include-rtools: false
- name: Install node (for Playwright, MECA)
uses: actions/setup-node@v4
with:
node-version: 20
- name: Install node dependencies
if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }}
run: yarn
working-directory: ./tests/integration/playwright
shell: bash
- name: Install Playwright Browsers
if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }}
run: npx playwright install --with-deps
working-directory: ./tests/integration/playwright
- name: Install MECA validator
if: ${{ runner.os != 'Windows' }}
run: npm install -g meca
- name: Set RENV_PATHS_ROOT
shell: bash
run: |
echo "RENV_PATHS_ROOT=${{ runner.temp }}/renv" >> $GITHUB_ENV
- name: Get R and OS version
id: get-version
run: |
cat("os-version=", sessionInfo()$running, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE)
cat("r-version=", R.Version()$version.string, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE)
shell: Rscript {0}
- name: Cache R packages
uses: actions/cache@v4
with:
path: ${{ env.RENV_PATHS_ROOT }}
key: ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2-${{ hashFiles('tests/renv.lock') }}
restore-keys: |
${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2-
save-always: ${{ inputs.extra-r-packages == '' }} # don't save cache if we have extra R packages
- name: Install missing system deps
if: runner.os == 'Linux'
run: |
sudo apt-get update -y
sudo apt-get install -y libcurl4-openssl-dev
sudo apt-get install -y libxml2-utils
sudo apt-get install -y libharfbuzz-dev libfribidi-dev
sudo apt-get install -y poppler-utils
- name: Restore R packages
working-directory: tests
run: |
if (!requireNamespace('renv', quietly = TRUE)) install.packages('renv')
renv::restore()
# Install dev versions for our testing
# Use r-universe to avoid github api calls
try(install.packages('rmarkdown', repos = c('https://rstudio.r-universe.dev', getOption('repos'))))
try(install.packages('knitr', repos = c('https://yihui.r-universe.dev', getOption('repos'))))
if ('${{ inputs.extra-r-packages }}' != '') {
cat(sprintf("::notice::Running with the following extra R packages for pak: %s\n", "${{ inputs.extra-r-packages }}"))
renv::install(strsplit("${{ inputs.extra-r-packages }}", split = ",")[[1]])
}
shell: Rscript {0}
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
- name: Install uv for Python
uses: astral-sh/setup-uv@v3
with:
version: "0.5.9"
enable-cache: true
cache-dependency-glob: "tests/uv.lock"
- name: Install Python and Dependencies
working-directory: tests
run: |
uv sync --frozen
- uses: ./.github/workflows/actions/quarto-dev
- name: Install Tinytex
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
quarto install tinytex
- name: Cache Typst packages
id: cache-typst
uses: ./.github/actions/cache-typst
- name: Install Chrome
uses: browser-actions/setup-chrome@latest
- name: Setup Julia
uses: julia-actions/setup-julia@v2
with:
version: "1.11"
- name: Cache Julia Packages
uses: julia-actions/cache@v2
- name: Restore Julia Packages
working-directory: tests
shell: bash
run: |
# Setup IJulia with the jupyter from the Python environment
# https://julialang.github.io/IJulia.jl/stable/manual/installation/
export JUPYTER=$(find $(dirname $(uv run --frozen which jupyter))/ -type f -name "jupyter.exe" -o -name "jupyter")
uv run --frozen julia --color=yes --project=. -e "import Pkg; Pkg.instantiate(); Pkg.build(\"IJulia\"); Pkg.precompile()"
echo "Julia Jupyter:"
uv run julia --project=. -e "import IJulia;println(IJulia.JUPYTER);println(IJulia.find_jupyter_subcommand(\"notebook\"))"
- name: Setup timing file for timed test
if: ${{ matrix.time-test == true }}
run: |
echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV"
# - name: Setup tmate session
# uses: mxschmitt/action-tmate@v3
- name: Run all Smoke Tests Windows
if: ${{ runner.os == 'Windows' && format('{0}', inputs.buckets) == '' && matrix.time-test == false }}
env:
# Useful as TinyTeX latest release is checked in run-test.sh
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ./run-tests.ps1
working-directory: tests
shell: pwsh
- name: Run all Smoke Tests Linux
if: ${{ runner.os != 'Windows' && format('{0}', inputs.buckets) == '' }}
env:
# Useful as TinyTeX latest release is checked in run-test.sh
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ./run-tests.sh
working-directory: tests
shell: bash
- name: Run Smoke Tests as a bucket on Linux
if: ${{ runner.os != 'Windows' && format('{0}', inputs.buckets) != '' }}
env:
# Useful as TinyTeX latest release is checked in run-test.sh
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
QUARTO_LOG_LEVEL: DEBUG
run: |
haserror=0
readarray -t my_array < <(echo '${{ inputs.buckets }}' | jq -rc '.[]')
for file in "${my_array[@]}"; do
echo ">>> ./run-tests.sh ${file}"
./run-tests.sh $file
status=$?
[ $status -eq 0 ] && echo ">>> No error in this test file" || haserror=1
done
[ $haserror -eq 0 ] && echo ">>> All tests passed" || exit 1
working-directory: tests
shell: bash
- name: Run Smoke Tests as a bucket on Windows
if: ${{ runner.os == 'Windows' && format('{0}', inputs.buckets) != '' }}
env:
# Useful as TinyTeX latest release is checked in run-test.sh
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
$haserror=$false
foreach ($file in ('${{ inputs.buckets }}' | ConvertFrom-Json)) {
Write-Host ">>> ./run-tests.ps1 ${file}"
./run-tests.ps1 $file
$status=$LASTEXITCODE
if ($status -eq 1) {
Write-Host ">>> Error found in test file"
$haserror=$true
} else {
Write-Host ">>> No error in this test file"
}
}
if ($haserror) {
Exit 1
} else {
Write-Host ">>> All tests have passed"
}
working-directory: tests
shell: pwsh
- name: Create Pull Request for new tests timing
if: matrix.time-test
id: cpr
uses: peter-evans/create-pull-request@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
add-paths: tests/timing-for-ci.txt
commit-message: |
timing for new tests [auto PR]
branch: updates/timing-for-ci
delete-branch: true
assignees: cderv
reviewers: cderv
title: |
[CI] Update timing file
body: |
This PR was created automatically with new timing test file for our parallel smoke tests.
Please review and merge to trigger a new build of the website.
- name: Check auto PR outputs
if: steps.cpr.outcome == 'success'
run: |
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
echo "Pull Request Action Performed - ${{ steps.cpr.outputs.pull-request-operation }}"
- name: Upload test timing file
uses: actions/upload-artifact@v4
if: matrix.time-test && ( failure() || cancelled())
with:
name: timed test file
path: tests/timing-for-ci.txt
- name: Save Typst cache
if: always()
uses: actions/cache/save@v4
with:
key: ${{ steps.cache-typst.outputs.cache-primary-key }}
path: ${{ steps.cache-typst.outputs.cache-path }}
- uses: actions/upload-artifact@v4
# PLaywright test only runs on Linux for now
if: ${{ !cancelled() && runner.os != 'Windows' }}
with:
name: playwright-report
path: ./tests/integration/playwright/playwright-report/
retention-days: 30