diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5aeb3709..b99c9fb1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,21 +57,18 @@ jobs: - name: Install pip dependencies run: | - pip install -e . - pip install -e .[mpi] - pip install -e .[compression] - pip install -e .[test] + pip install -e .[compression,mpi,test] - name: Run serial tests - run: pytest --doctest-modules caput/ + run: pytest --doctest-modules . - name: Run parallel tests run: | - mpirun --oversubscribe -np 4 pytest caput/tests/test_memh5_parallel.py - mpirun --oversubscribe -np 4 pytest caput/tests/test_mpiarray.py - mpirun -np 1 pytest caput/tests/test_selection_parallel.py - mpirun --oversubscribe -np 2 pytest caput/tests/test_selection_parallel.py - mpirun --oversubscribe -np 4 pytest caput/tests/test_selection_parallel.py + mpirun --oversubscribe -np 4 pytest tests/test_memh5_parallel.py + mpirun --oversubscribe -np 4 pytest tests/test_mpiarray.py + mpirun -np 1 pytest tests/test_selection_parallel.py + mpirun --oversubscribe -np 2 pytest tests/test_selection_parallel.py + mpirun --oversubscribe -np 4 pytest tests/test_selection_parallel.py build-docs: diff --git a/caput/tests/test_pipeline.py b/caput/tests/test_pipeline.py deleted file mode 100644 index 22d7b28b..00000000 --- a/caput/tests/test_pipeline.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Test running the caput.pipeline.""" - -from caput.tests import conftest - - -def test_pipeline(): - """Test running a very simple pipeline.""" - result = conftest.run_pipeline() - print(result.output) - assert result.exit_code == 0 - - -def test_pipeline_multiple_outputs(): - """Test running a very simple pipeline with a multi-output task.""" - result = conftest.run_pipeline(configstr=conftest.multi_eggs_pipeline_conf) - print(result.output) - assert result.exit_code == 0 diff --git a/doc/conf.py b/doc/conf.py index 7eaab557..3bb71d9d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -21,7 +21,7 @@ import caput # Mock up modules missing on readthedocs. -from mock import Mock as MagicMock +from unittest.mock import Mock as MagicMock class Mock(MagicMock): diff --git a/caput/tests/README b/tests/README similarity index 100% rename from caput/tests/README rename to tests/README diff --git a/caput/tests/__init__.py b/tests/__init__.py similarity index 100% rename from caput/tests/__init__.py rename to tests/__init__.py diff --git a/caput/tests/conftest.py b/tests/conftest.py similarity index 64% rename from caput/tests/conftest.py rename to tests/conftest.py index eb590cd4..91d5301b 100644 --- a/caput/tests/conftest.py +++ b/tests/conftest.py @@ -6,9 +6,9 @@ import numpy as np import pytest -from ..pipeline import PipelineStopIteration, TaskBase, IterBase -from ..scripts.runner import cli -from .. import config, fileformats, mpiutil +from caput.pipeline import PipelineStopIteration, TaskBase, IterBase +from caput.scripts.runner import cli +from caput import config, fileformats, mpiutil @pytest.fixture(scope="session") @@ -106,16 +106,19 @@ def write_output(self, filename, output, file_format=None, **kwargs): raise NotImplementedError() -eggs_pipeline_conf = """ +@pytest.fixture +def run_pipeline(): + """Provides the `run_pipeline` function which will run the pipeline.""" + eggs_pipeline_conf = """ --- pipeline: tasks: - - type: caput.tests.conftest.PrintEggs + - type: tests.conftest.PrintEggs params: eggs_params - - type: caput.tests.conftest.GetEggs + - type: tests.conftest.GetEggs params: eggs_params out: egg - - type: caput.tests.conftest.CookEggs + - type: tests.conftest.CookEggs params: cook_params in: egg eggs_params: @@ -124,55 +127,37 @@ def write_output(self, filename, output, file_format=None, **kwargs): style: 'fried' """ -multi_eggs_pipeline_conf = """ ---- -pipeline: - tasks: - - type: caput.tests.conftest.GetEggs - params: eggs_params - out: [color, egg] - - type: caput.tests.conftest.CookEggs - params: cook_params - in: egg - - type: caput.tests.conftest.CookEggs - params: cook_params - in: color -eggs_params: - eggs: [['green', 'duck'], ['blue', 'ostrich']] -cook_params: - style: 'fried' -""" - + def _run_pipeline(parameters=None, configstr=eggs_pipeline_conf): + """Run `caput.scripts.runner run` with given parameters and config. -def run_pipeline(parameters=None, configstr=eggs_pipeline_conf): - """Run `caput.scripts.runner run` with given parameters and config. + Parameters + ---------- + parameters : List[str] + Parameters to pass to the cli, for example `["--profile"]` (see `--help`). + configstr : str + YAML string to use as a config. This function will write it to a file that is then passed to the cli. - Parameters - ---------- - parameters : List[str] - Parameters to pass to the cli, for example `["--profile"]` (see `--help`). - configstr : str - YAML string to use as a config. This function will write it to a file that is then passed to the cli. + Returns + ------- + result : `click.testing.Result` + Holds the captured result. Try accessing e.g. `result.exit_code`, `result.output`. + """ + with tempfile.NamedTemporaryFile("w+") as configfile: + configfile.write(configstr) + configfile.flush() + from click.testing import CliRunner - Returns - ------- - result : `click.testing.Result` - Holds the captured result. Try accessing e.g. `result.exit_code`, `result.output`. - """ - with tempfile.NamedTemporaryFile("w+") as configfile: - configfile.write(configstr) - configfile.flush() - from click.testing import CliRunner + runner = CliRunner() + if parameters is None: + return runner.invoke(cli, ["run", configfile.name]) + else: + return runner.invoke(cli, ["run", *parameters, configfile.name]) - runner = CliRunner() - if parameters is None: - return runner.invoke(cli, ["run", configfile.name]) - else: - return runner.invoke(cli, ["run", *parameters, configfile.name]) + return _run_pipeline @pytest.fixture -def h5_file(): +def h5_file(rm_all_files): """Provides a file name and removes all files/dirs with the same prefix later.""" fname = "tmp_test_memh5.h5" yield fname @@ -180,7 +165,7 @@ def h5_file(): @pytest.fixture -def zarr_file(): +def zarr_file(rm_all_files): """Provides a directory name and removes all files/dirs with the same prefix later.""" fname = "tmp_test_memh5.zarr" yield fname @@ -188,7 +173,7 @@ def zarr_file(): @pytest.fixture -def h5_file_distributed(): +def h5_file_distributed(rm_all_files): """Provides a file name and removes all files/dirs with the same prefix later.""" fname = "tmp_test_memh5_distributed.h5" yield fname @@ -197,7 +182,7 @@ def h5_file_distributed(): @pytest.fixture -def zarr_file_distributed(): +def zarr_file_distributed(rm_all_files): """Provides a directory name and removes all files/dirs with the same prefix later.""" fname = "tmp_test_memh5.zarr" yield fname @@ -205,8 +190,14 @@ def zarr_file_distributed(): rm_all_files(fname) -def rm_all_files(file_name): - """Remove all files and directories starting with `file_name`.""" - file_names = glob.glob(file_name + "*") - for fname in file_names: - fileformats.remove_file_or_dir(fname) +@pytest.fixture +def rm_all_files(): + """Provides the `rm_all_files` function.""" + + def _rm_all_files(file_name): + """Remove all files and directories starting with `file_name`.""" + file_names = glob.glob(file_name + "*") + for fname in file_names: + fileformats.remove_file_or_dir(fname) + + return _rm_all_files diff --git a/caput/tests/test_config.py b/tests/test_config.py similarity index 100% rename from caput/tests/test_config.py rename to tests/test_config.py diff --git a/caput/tests/test_interferometry.py b/tests/test_interferometry.py similarity index 100% rename from caput/tests/test_interferometry.py rename to tests/test_interferometry.py diff --git a/caput/tests/test_lint.py b/tests/test_lint.py similarity index 92% rename from caput/tests/test_lint.py rename to tests/test_lint.py index 0fc27768..e372b9f0 100644 --- a/caput/tests/test_lint.py +++ b/tests/test_lint.py @@ -23,16 +23,16 @@ def simple_config(): "pipeline": { "tasks": [ { - "type": "caput.tests.test_lint.DoNothing", + "type": "tests.test_lint.DoNothing", "out": "out1", }, { - "type": "caput.tests.test_lint.DoNothing", + "type": "tests.test_lint.DoNothing", "out": "out2", "in": "out1", }, { - "type": "caput.tests.test_lint.DoNothing2", + "type": "tests.test_lint.DoNothing2", "in": "out2", "out": "out3", "requires": "out1", diff --git a/caput/tests/test_memh5.py b/tests/test_memh5.py similarity index 100% rename from caput/tests/test_memh5.py rename to tests/test_memh5.py diff --git a/caput/tests/test_memh5_parallel.py b/tests/test_memh5_parallel.py similarity index 100% rename from caput/tests/test_memh5_parallel.py rename to tests/test_memh5_parallel.py diff --git a/caput/tests/test_metadata.py b/tests/test_metadata.py similarity index 100% rename from caput/tests/test_metadata.py rename to tests/test_metadata.py diff --git a/caput/tests/test_misc.py b/tests/test_misc.py similarity index 100% rename from caput/tests/test_misc.py rename to tests/test_misc.py diff --git a/caput/tests/test_moving_weighted_median.py b/tests/test_moving_weighted_median.py similarity index 100% rename from caput/tests/test_moving_weighted_median.py rename to tests/test_moving_weighted_median.py diff --git a/caput/tests/test_mpiarray.py b/tests/test_mpiarray.py similarity index 100% rename from caput/tests/test_mpiarray.py rename to tests/test_mpiarray.py diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 00000000..5fd16bcb --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,35 @@ +"""Test running the caput.pipeline.""" + + +def test_pipeline(run_pipeline): + """Test running a very simple pipeline.""" + result = run_pipeline() + print(result.output) + assert result.exit_code == 0 + + +def test_pipeline_multiple_outputs(run_pipeline): + """Test running a very simple pipeline with a multi-output task.""" + + multi_eggs_pipeline_conf = """ +--- +pipeline: + tasks: + - type: tests.conftest.GetEggs + params: eggs_params + out: [color, egg] + - type: tests.conftest.CookEggs + params: cook_params + in: egg + - type: tests.conftest.CookEggs + params: cook_params + in: color +eggs_params: + eggs: [['green', 'duck'], ['blue', 'ostrich']] +cook_params: + style: 'fried' +""" + + result = run_pipeline(configstr=multi_eggs_pipeline_conf) + print(result.output) + assert result.exit_code == 0 diff --git a/caput/tests/test_psutilprofiler.py b/tests/test_psutilprofiler.py similarity index 58% rename from caput/tests/test_psutilprofiler.py rename to tests/test_psutilprofiler.py index 0764c36c..92e10cb4 100755 --- a/caput/tests/test_psutilprofiler.py +++ b/tests/test_psutilprofiler.py @@ -1,10 +1,8 @@ """Test running the caput.pipeline.""" -from caput.tests import conftest - -def test_pipeline(): +def test_pipeline(run_pipeline): """Test profiling a very simple pipeline.""" - result = conftest.run_pipeline(["--psutil"]) + result = run_pipeline(["--psutil"]) print(result.output) assert result.exit_code == 0 diff --git a/caput/tests/test_selection.py b/tests/test_selection.py similarity index 94% rename from caput/tests/test_selection.py rename to tests/test_selection.py index 859842c2..f133149e 100644 --- a/caput/tests/test_selection.py +++ b/tests/test_selection.py @@ -6,7 +6,6 @@ from caput.memh5 import MemGroup from caput import fileformats -from caput.tests.conftest import rm_all_files fsel = slice(1, 8, 2) isel = slice(1, 4) @@ -16,7 +15,7 @@ @pytest.fixture -def h5_file_select(datasets, h5_file): +def h5_file_select(datasets, h5_file, rm_all_files): """Provides an HDF5 file with some content for testing.""" container = MemGroup() container.create_dataset("dset1", data=datasets[0].view()) @@ -27,7 +26,7 @@ def h5_file_select(datasets, h5_file): @pytest.fixture -def zarr_file_select(datasets, zarr_file): +def zarr_file_select(datasets, zarr_file, rm_all_files): """Provides a Zarr file with some content for testing.""" container = MemGroup() container.create_dataset("dset1", data=datasets[0].view()) diff --git a/caput/tests/test_selection_parallel.py b/tests/test_selection_parallel.py similarity index 97% rename from caput/tests/test_selection_parallel.py rename to tests/test_selection_parallel.py index 887c4688..1ef65fe5 100644 --- a/caput/tests/test_selection_parallel.py +++ b/tests/test_selection_parallel.py @@ -10,14 +10,13 @@ from caput import mpiutil, mpiarray, fileformats from caput.memh5 import MemGroup -from caput.tests.conftest import rm_all_files comm = MPI.COMM_WORLD @pytest.fixture -def container_on_disk(datasets, file_name, file_format): +def container_on_disk(datasets, file_name, file_format, rm_all_files): """Prepare a file for the select_parallel tests.""" if comm.rank == 0: m1 = mpiarray.MPIArray.wrap(datasets[0], axis=0, comm=MPI.COMM_SELF) diff --git a/caput/tests/test_time.py b/tests/test_time.py similarity index 100% rename from caput/tests/test_time.py rename to tests/test_time.py diff --git a/caput/tests/test_tod.py b/tests/test_tod.py similarity index 100% rename from caput/tests/test_tod.py rename to tests/test_tod.py diff --git a/caput/tests/test_tools.py b/tests/test_tools.py similarity index 100% rename from caput/tests/test_tools.py rename to tests/test_tools.py diff --git a/caput/tests/test_truncate.py b/tests/test_truncate.py similarity index 100% rename from caput/tests/test_truncate.py rename to tests/test_truncate.py