test: testing common wheels (#2031)

Co-authored-by: Hofer-Julian <[email protected]>

.github/workflows/rust.yml

@@ -417,3 +417,9 @@ jobs:
     needs:
       - build
     uses: ./.github/workflows/test_downstream.yml
+
+  test_common_wheels:
+    name: "Test installation of common wheels"
+    needs:
+      - build
+    uses: ./.github/workflows/test_common_wheels.yml

.github/workflows/test_common_wheels.yml

@@ -0,0 +1,77 @@
+name: "Test common wheel files for installation with pixi"
+
+on:
+  workflow_call:
+
+jobs:
+  test_common_wheels:
+    name: ${{ matrix.arch.name }} - Test Installation of Common Wheels
+    runs-on: ${{ matrix.arch.os }}
+    env:
+      TARGET_RELEASE: "${{ github.workspace }}/.pixi/target/release"
+      LOGS_DIR: "${{ github.workspace }}/tests/wheel_tests/.logs"
+      SUMMARY_FILE: "${{ github.workspace }}/tests/wheel_tests/.summary.md"
+      PYTHONIOENCODING: utf-8
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          # Linux
+          - {
+              target: x86_64-unknown-linux-musl,
+              os: 8core_ubuntu_latest_runner,
+              name: "Linux",
+            }
+          # MacOS
+          - { target: x86_64-apple-darwin, os: macos-13, name: "MacOS-x86" }
+          - { target: aarch64-apple-darwin, os: macos-14, name: "MacOS-Arm" } # macOS-14 is the ARM chipset
+          # Windows
+          - {
+              target: x86_64-pc-windows-msvc,
+              os: windows-latest,
+              extension: .exe,
+              name: "Windows",
+            }
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Download binary from build
+        uses: actions/download-artifact@v4
+        with:
+          name: pixi-${{ matrix.arch.target }}${{ matrix.arch.extension }}
+          path: pixi_bin
+      - name: Debug
+        run: |
+          pwd
+      - name: Create Directory and Move Executable to TARGET_RELEASE
+        if: matrix.arch.name != 'Windows'
+        run: |
+          mkdir -p ${{ env.TARGET_RELEASE }}
+          mv pixi_bin/pixi-${{ matrix.arch.target }} ${{ env.TARGET_RELEASE }}/pixi
+          chmod a+x ${{ env.TARGET_RELEASE }}/pixi
+      - name: Create Directory and Move Executable to TARGET_RELEASE
+        if: matrix.arch.name == 'Windows' && matrix.arch.target == 'x86_64-pc-windows-msvc'
+        run: |
+          New-Item -ItemType Directory -Force -Path "${{ env.TARGET_RELEASE }}"
+          Move-Item -Path "pixi_bin/pixi-${{ matrix.arch.target }}${{ matrix.arch.extension }}" -Destination "${{ env.TARGET_RELEASE }}/pixi.exe"
+        shell: pwsh
+      - name: Test common wheels
+        run: ${{ env.TARGET_RELEASE }}/pixi${{ matrix.arch.extension }} run test-common-wheels-ci
+      - name: Write .summary.md to Github Summary
+        if: ${{ matrix.arch.name != 'Windows' && always() }}
+        shell: bash
+        run: |
+          cat ${{ env.SUMMARY_FILE }} >> $GITHUB_STEP_SUMMARY
+      - name: Write .summary.md to GitHub Summary (Windows)
+        if: ${{ matrix.arch.name == 'Windows' && always() }}
+        shell: pwsh
+        run: |
+          $resolvedPath = Resolve-Path $env:SUMMARY_FILE
+          Get-Content $resolvedPath | Out-File -Append -FilePath $env:GITHUB_STEP_SUMMARY
+      - name: Upload Logs
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: wheel-tests-logs-${{ matrix.arch.name }}
+          include-hidden-files: true
+          path: ${{ env.LOGS_DIR }}

pixi.toml

@@ -12,7 +12,7 @@ channels = ["https://fast.prefix.dev/conda-forge"]
 platforms = ["linux-64", "win-64", "osx-64", "osx-arm64"]
 
 [dependencies]
-python = ">=3.12.3,<4"
+python = "3.12.*"
 
 [tasks]
 build = "cargo build --release"
@@ -31,9 +31,17 @@ mypy = ">=1.11,<1.12"
 # For detecting cpu cores with pytest-xdist
 psutil = ">=6.0.0,<7"
 # For running tests in parallel, use this instead of regular pytest
+filelock = ">=3.16.0,<4"
+pytest-rerunfailures = ">=14.0,<15"
 pytest-xdist = ">=3.6.1,<4"
+rich = ">=13.7.1,<14"
+toml = ">=0.10.2,<0.11"
 
 [feature.pytest.tasks]
+test-common-wheels-ci = { cmd = "pytest -n logical tests/wheel_tests/" }
+test-common-wheels-dev = { cmd = "pytest -n logical tests/wheel_tests/", depends-on = [
+  "build",
+] }
 test-integration-ci = "pytest -n logical tests/integration"
 test-integration-dev = { cmd = "pytest -n logical tests/integration", depends-on = [
   "build",

tests/wheel_tests/.gitignore

@@ -0,0 +1,4 @@
+.wheel_test_results.toml
+.logs/**
+.summary.md
+.wheel_test_results.lock

tests/wheel_tests/conftest.py

@@ -0,0 +1,26 @@
+from generate_summaries import terminal_summary, markdown_summary
+from helpers import setup_stdout_stderr_logging
+
+
+def pytest_configure(config):
+    setup_stdout_stderr_logging()
+
+
+def pytest_addoption(parser):
+    # Used to override the default path to the pixi executable
+    parser.addoption("--pixi-exec", action="store", help="Path to the pixi executable")
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+    """
+    At the end of the test session, generate a summary report.
+    """
+    terminal_summary()
+
+
+def pytest_sessionfinish(session, exitstatus):
+    """
+    At the end of the test session, generate a `.summary.md` report. That contains the
+    same information as the terminal summary.
+    """
+    markdown_summary()

tests/wheel_tests/generate_summaries.py

@@ -0,0 +1,116 @@
+from record_results import RESULTS_FILE
+import toml
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+from rich.panel import Panel
+from pathlib import Path
+
+from read_wheels import read_wheel_file
+
+
+def terminal_summary():
+    # Read aggregated results from the shared file
+    results_file = RESULTS_FILE
+    if not results_file.exists():
+        print("Error: No test results found.")
+        return
+
+    with results_file.open("r") as f:
+        results = toml.load(f)["results"]
+
+    packages = read_wheel_file()
+
+    console = Console()
+    table = Table(title="Test Results", show_header=True, header_style="bold magenta")
+    table.add_column("Test Name", style="dim")
+    table.add_column("Outcome", justify="right")
+    table.add_column("Duration (s)", justify="right")
+    table.add_column("Error Details")
+
+    # Populate the table with collected results
+    names = []
+    for result in sorted(results, key=lambda r: r["name"]):
+        outcome_color = "green" if result["outcome"] == "passed" else "red"
+        error_details = result["longrepr"] if result["outcome"] == "failed" else ""
+        table.add_row(
+            Text(result["name"]),
+            Text(result["outcome"], style=outcome_color),
+            f"{result['duration']:.2f}",
+            error_details,
+        )
+        # Record name
+        names.append(result["name"])
+
+    for package in packages:
+        if package.to_add_cmd() not in names:
+            table.add_row(
+                Text(package.to_add_cmd()),
+                Text("N/A", style="dim"),
+                Text("N/A", style="dim"),
+                Text("N/A", style="dim"),
+            )
+
+    # Display the table in the terminal
+    console.print(table)
+
+    # Add a summary box with instructions
+    summary_text = (
+        "[bold]Summary:[/bold]\n\n"
+        f"- Total tests run: {len(results)}\n"
+        f"- Passed: {sum(1 for r in results if r['outcome'] == 'passed')}\n"
+        f"- Failed: {sum(1 for r in results if r['outcome'] == 'failed')}\n\n"
+        "To filter tests by a specific wheel, use the command:\n"
+        "[bold green]pytest -k '<pixi_add_cmd>'[/]\n\n"
+        "Replace [bold]<pixi_add_com>[/] with the desired wheel's name to run only tests for that wheel.\n"
+        r'E.g use [magenta] pixi r test-common-wheels-dev -k "jax\[cuda12]"[/] to run tests for the [bold]jax\[cuda12][/] wheel.'
+        "\n\n"
+        "[bold yellow]Note:[/]\n"
+        "Any [italic]failed[/] tests will have recorded their output to the [bold].log/[/] directory, which"
+        " resides next to to `wheels.toml` file.\n"
+    )
+
+    # Create a Rich panel (box) for the summary text
+    summary_panel = Panel(
+        summary_text, title="Test Debrief", title_align="left", border_style="bright_blue"
+    )
+
+    # Display the summary box in the terminal
+    console.print(summary_panel)
+
+
+def markdown_summary():
+    if not RESULTS_FILE.exists():
+        return
+
+    summary_file = Path(__file__).parent / ".summary.md"
+    with summary_file.open("w") as f:
+        # Read the RESULTS_FILE and generate a markdown summary
+        f.write("# Test Summary\n\n")
+        f.write("""
+This document contains a summary of the test results for the wheels in the `wheels.toml` file.
+You can use the following command, in the pixi repository, to filter tests by a specific wheel:
+```bash
+pixi r test-common-wheels -k "<pixi_add_cmd>"
+# E.g
+pixi r test-common-wheels-dev -k "jax[cuda12]"
+```
+
+""")
+        f.write("## Test Results\n\n")
+        f.write("\n")
+        f.write("| Test Name | Outcome | Duration (s) | Error Details |\n")
+        f.write("| :--- | ---: | ---: | --- |\n")
+
+        results_file = RESULTS_FILE
+        with results_file.open("r") as r:
+            results = toml.load(r)["results"]
+            for result in results:
+                outcome = (
+                    '<span style="color: green">Passed</span>'
+                    if result["outcome"] == "passed"
+                    else '<span style="color: red">Failed</span>'
+                )
+                error_details = result["longrepr"] if result["outcome"] == "failed" else ""
+                f.write(f"|{result["name"]}|{outcome}|{result['duration']:.2f}|{error_details}|\n")
+        f.write("\n")

tests/wheel_tests/helpers.py

@@ -0,0 +1,61 @@
+import os
+import subprocess
+import pathlib
+from typing import Any
+import toml
+
+StrPath = str | os.PathLike[str]
+LOG_DIR = pathlib.Path(__file__).parent / ".logs"
+
+
+def run(args: list[StrPath], cwd: StrPath | None = None) -> None:
+    """
+    Run a subprocess and check the return code
+    """
+    proc: subprocess.CompletedProcess[bytes] = subprocess.run(
+        args, cwd=cwd, capture_output=True, check=False
+    )
+    proc.check_returncode()
+
+
+def add_system_requirements(manifest_path: pathlib.Path, system_requirements: dict[str, Any]):
+    """
+    Add system requirements to the manifest file
+    add something like this:
+        [system-requirements]
+        libc = { family = "glibc", version = "2.17" }
+    to the manifest file.
+    """
+    with manifest_path.open("r") as f:
+        manifest = toml.load(f)
+    manifest["system-requirements"] = system_requirements
+    with manifest_path.open("w") as f:
+        toml.dump(manifest, f)
+
+
+def setup_stdout_stderr_logging():
+    """
+    Set up the logging directory
+    """
+    if not LOG_DIR.exists():
+        LOG_DIR.mkdir()
+    for file in LOG_DIR.iterdir():
+        file.unlink()
+
+
+def log_called_process_error(name: str, err: subprocess.CalledProcessError, std_err_only=False):
+    """
+    Log the output of a subprocess that failed
+    has the option to log only the stderr
+    """
+    if not LOG_DIR.exists():
+        raise RuntimeError("Call setup_stdout_stderr_logging before logging")
+
+    std_out_log = LOG_DIR / f"{name}.stdout"
+    std_err_log = LOG_DIR / f"{name}.stderr"
+    if err.returncode != 0:
+        if not std_err_only:
+            with std_out_log.open("w", encoding="utf-8") as f:
+                f.write(err.stdout.decode("uft-8"))
+        with std_err_log.open("w", encoding="utf-8") as f:
+            f.write(err.stderr.decode("utf-8"))