From 9466a15aa132a5524a3b2de657fdfec16dffd974 Mon Sep 17 00:00:00 2001 From: Lynne Jones Date: Thu, 2 Nov 2023 18:06:11 -0700 Subject: [PATCH 1/2] Update tests; consolidate pypi build and publish --- .github/workflows/cache.yaml | 14 ++-- .github/workflows/python_tests.yaml | 83 ------------------- .../{build_pypi.yaml => test_and_build.yaml} | 75 ++++++++++------- 3 files changed, 53 insertions(+), 119 deletions(-) delete mode 100644 .github/workflows/python_tests.yaml rename .github/workflows/{build_pypi.yaml => test_and_build.yaml} (53%) diff --git a/.github/workflows/cache.yaml b/.github/workflows/cache.yaml index d61c60cb..fe0f42fb 100644 --- a/.github/workflows/cache.yaml +++ b/.github/workflows/cache.yaml @@ -29,13 +29,12 @@ jobs: id: rs-install shell: bash -l {0} run: | - export RUBIN_SIM_DATA_DIR=~/rubin_sim_data mamba install --quiet rubin_sim - python -c "import rubin_sim; print(rubin_sim.__version__)" > ${{ github.workspace }}/rs_version - echo `cat ${{ github.workspace }}/rs_version` + mamba list rubin-sim | grep -v "#" | awk '{print $2}' > ${{ github.workspace }}/rs_version + echo "rs-version" `cat ${{ github.workspace }}/rs_version` echo "rs-version=`cat ${{ github.workspace }}/rs_version`" >> $GITHUB_OUTPUT - - name: Cache rubin-sim-data + - name: Access rubin-sim-data cache id: cache-rs uses: actions/cache@v3 env: @@ -47,18 +46,17 @@ jobs: ${{ env.cache-name }}-v - if: ${{ steps.cache-rs.outputs.cache-hit == 'true' }} - name: Hit cache? List contents - continue-on-error: true + name: List cache contents. shell: bash -l {0} run: | export RUBIN_SIM_DATA_DIR=~/rubin_sim_data - echo $RUBIN_SIM_DATA_DIR (contents) + echo $RUBIN_SIM_DATA_DIR contents ls $RUBIN_SIM_DATA_DIR echo "__contents of versions.txt__" cat $RUBIN_SIM_DATA_DIR/versions.txt - if: ${{ steps.cache-rs.outputs.cache-hit != 'true' }} - name: Download data + name: Download data. shell: bash -l {0} run: | export RUBIN_SIM_DATA_DIR=~/rubin_sim_data diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml deleted file mode 100644 index 5e1f512b..00000000 --- a/.github/workflows/python_tests.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: Run Unit Tests - -on: - # Trigger the workflow on pull request to main, or by hand - # Trigger the workflow on push (to main) or pull request - push: - branches: - - main - pull_request: - branches: - - main - workflow_dispatch: - -jobs: - Tests: - name: Run Tests (${{ matrix.python-version }}, ${{ matrix.os }}) - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest", "macos-latest"] - python-version: ["3.11"] - steps: - - uses: actions/checkout@v3 - - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: ${{ matrix.python-version }} - auto-update-conda: true - channels: conda-forge,defaults - miniforge-variant: Mambaforge - use-mamba: true - channel-priority: strict - show-channel-urls: true - - - name: configure conda and install requirements - shell: bash -l {0} - run: | - mamba config --set always_yes yes - mamba install --quiet --file=requirements.txt - mamba install --quiet --file=test-requirements.txt - - - name: download rubin_sim_data components needed for unit tests - shell: bash -l {0} - run: | - export RUBIN_SIM_DATA_DIR=${{ github.workspace }}/data_dir - mkdir $RUBIN_SIM_DATA_DIR - cd $RUBIN_SIM_DATA_DIR - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/tests_2022_10_18.tgz - tar -xf tests_2022_10_18.tgz - echo "tests_2022_10_18.tgz complete" - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/throughputs_2023_09_07.tgz - tar -xf throughputs_2023_09_07.tgz - echo "throughputs_2023_09_07.tgz complete" - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/site_models_2023_03_28.tgz - tar -xf site_models_2023_03_28.tgz - echo "site_models_2023_03_28.tgz complete" - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/scheduler_2023_09_22.tgz - tar -xf scheduler_2023_09_22.tgz - echo "scheduler_2023_09_22.tgz complete" - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/skybrightness_pre_2023_10_17.tgz - tar -xf skybrightness_pre_2023_10_17.tgz - echo "skybrightness_pre_2023_10_17.tgz complete" - - - name: install schedview - shell: bash -l {0} - run: | - echo `pwd` - python -m pip install . - - - name: conda list - shell: bash -l {0} - run: conda list - - - name: run unit tests - shell: bash -l {0} - run: | - export RUBIN_SIM_DATA_DIR=${{ github.workspace }}/data_dir - pytest -r a -v --cov=schedview --cov=tests --cov-report=xml --cov-report=term --cov-branch - - - name: Upload coverage to codecov - uses: codecov/codecov-action@v2 - with: - file: coverage.xml diff --git a/.github/workflows/build_pypi.yaml b/.github/workflows/test_and_build.yaml similarity index 53% rename from .github/workflows/build_pypi.yaml rename to .github/workflows/test_and_build.yaml index fa5a8232..7d28edcf 100644 --- a/.github/workflows/build_pypi.yaml +++ b/.github/workflows/test_and_build.yaml @@ -1,18 +1,28 @@ -name: Build and Publish PyPI +name: Run Unit Tests on: + # Run job at the end of each day + schedule: + - cron: "0 0 * * *" + # Run job on push to main push: - tags: - - "*" + branches: + - main + # Run job on pull request to main + pull_request: + branches: + - main + # Run job manually on request + workflow_dispatch: jobs: tests: - name: Check Tests (${{ matrix.python-version }}, ${{ matrix.os }}) + name: Run Tests (${{ matrix.python-version }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] + os: ["ubuntu-latest", "macos-latest"] python-version: ["3.11"] steps: - uses: actions/checkout@v3 @@ -27,39 +37,48 @@ jobs: show-channel-urls: true - name: configure conda and install requirements + id: rs-install shell: bash -l {0} run: | - mamba config --set always_yes yes mamba install --quiet --file=requirements.txt mamba install --quiet --file=test-requirements.txt + mamba list rubin-sim | grep -v "#" | awk '{print $2}' > ${{ github.workspace }}/rs_version + echo "rs-version" `cat ${{ github.workspace }}/rs_version` + echo "rs-version=`cat ${{ github.workspace }}/rs_version`" >> $GITHUB_OUTPUT - - name: download rubin_sim_data components needed for unit tests + - name: Access rubin-sim-data cache + id: cache-rs + uses: actions/cache@v3 + env: + cache-name: cached-rubin-sim-data + with: + path: ~/rubin_sim_data + key: ${{ env.cache-name }}-v${{ steps.rs-install.outputs.rs-version }} + restore-keys: | + ${{ env.cache-name }}-v + + - if: ${{ steps.cache-rs.outputs.cache-hit == 'true' }} + name: List cache contents. + shell: bash -l {0} + run: | + export RUBIN_SIM_DATA_DIR=~/rubin_sim_data + echo $RUBIN_SIM_DATA_DIR + ls $RUBIN_SIM_DATA_DIR + echo "__contents of versions.txt__" + cat $RUBIN_SIM_DATA_DIR/versions.txt + + - if: ${{ steps.cache-rs.outputs.cache-hit != 'true' }} + name: Download data. shell: bash -l {0} run: | - export RUBIN_SIM_DATA_DIR=${{ github.workspace }}/data_dir - mkdir $RUBIN_SIM_DATA_DIR - cd $RUBIN_SIM_DATA_DIR - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/tests_2022_10_18.tgz - tar -xf tests_2022_10_18.tgz - echo "tests_2022_10_18.tgz complete" - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/throughputs_2023_09_07.tgz - tar -xf throughputs_2023_09_07.tgz - echo "throughputs_2023_09_07.tgz complete" - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/site_models_2023_03_28.tgz - tar -xf site_models_2023_03_28.tgz - echo "site_models_2023_03_28.tgz complete" - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/scheduler_2023_09_22.tgz - tar -xf scheduler_2023_09_22.tgz - echo "scheduler_2023_09_22.tgz complete" - wget -q https://s3df.slac.stanford.edu/data/rubin/sim-data/rubin_sim_data/skybrightness_pre_2023_09_19.tgz - tar -xf skybrightness_pre_2023_09_19.tgz - echo "skybrightness_pre_2023_09_19.tgz complete" + export RUBIN_SIM_DATA_DIR=~/rubin_sim_data + rs_download_data --force --dirs scheduler,site_models,skybrightness_pre,throughputs,tests --tdqm_disable - name: install schedview shell: bash -l {0} run: | echo `pwd` - python -m pip install . + python -m pip install . --no-deps - name: conda list shell: bash -l {0} @@ -68,7 +87,7 @@ jobs: - name: run unit tests shell: bash -l {0} run: | - export RUBIN_SIM_DATA_DIR=${{ github.workspace }}/data_dir + export RUBIN_SIM_DATA_DIR=~/rubin_sim_data pytest -r a -v --cov=schedview --cov=tests --cov-report=xml --cov-report=term --cov-branch - name: Upload coverage to codecov @@ -107,4 +126,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ - password: ${{ secrets.SP_PYPI_UPLOADS }} + password: ${{ secrets.SP_PYPI_UPLOADS }} \ No newline at end of file From 558ce3cef58e0771a408dbcfc563c22d6e360ca5 Mon Sep 17 00:00:00 2001 From: Lynne Jones Date: Fri, 3 Nov 2023 14:00:18 -0700 Subject: [PATCH 2/2] Change workflow name --- .github/workflows/test_and_build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_and_build.yaml b/.github/workflows/test_and_build.yaml index 7d28edcf..a13a2124 100644 --- a/.github/workflows/test_and_build.yaml +++ b/.github/workflows/test_and_build.yaml @@ -1,4 +1,4 @@ -name: Run Unit Tests +name: Run CI on: # Run job at the end of each day