Skip to content

Commit

Permalink
UPDATE tests
Browse files Browse the repository at this point in the history
  • Loading branch information
matbun committed Apr 30, 2024
1 parent 8fb77ec commit 1ef12e7
Show file tree
Hide file tree
Showing 10 changed files with 129 additions and 36 deletions.
30 changes: 29 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,35 @@ pip install -e .[dev]

#### Test with `pytest`

To run tests on itwinai package:
Do this only if you are a developer wanting to test your code with pytest.

First, you need to create virtual environments both for torch and tensorflow.
For instance, you can use:

```bash
make torch-cpu
make make tf-2.13-cpu
```

To select the name of the torch and tf environments you can set the following
environment variables, which allow to run the tests in environments with
custom names which are different from `.venv-pytorch` and `.venv-tf`.

```bash
export TORCH_ENV="my_torch_env"
export TF_ENV="my_tf_env"
```

Functional tests (marked with `pytest.mark.functional`) will be executed under
`/tmp/pytest` location to guarantee they are run in a clean environment.

To run functional tests use:

```bash
pytest -v tests/ -m "functional"
```

To run all tests on itwinai package:

```bash
# Activate env
Expand Down
24 changes: 22 additions & 2 deletions tests/use-cases/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from typing import Callable
import pytest
import subprocess
import random
import string


FNAMES = [
Expand All @@ -10,6 +12,24 @@
]


def rnd_string(len: int = 26):
return ''.join(random.sample(string.ascii_lowercase, len))


@pytest.fixture
def tmp_test_dir():
root = '/tmp/pytest'
os.makedirs(root, exist_ok=True)
test_dir = os.path.join(root, rnd_string())
while os.path.exists(test_dir):
test_dir = os.path.join(root, rnd_string())
os.makedirs(test_dir, exist_ok=True)

yield test_dir

# Optional: remove dir here...


@pytest.fixture
def torch_env() -> str:
"""
Expand All @@ -21,7 +41,7 @@ def torch_env() -> str:
env_p = './.venv-pytorch'
else:
env_p = os.environ.get('TORCH_ENV')
return os.path.join(os.getcwd(), env_p)
return os.path.abspath(env_p)


@pytest.fixture
Expand All @@ -35,7 +55,7 @@ def tf_env() -> str:
env_p = './.venv-tf'
else:
env_p = os.environ.get('TF_ENV')
return os.path.join(os.getcwd(), env_p)
return os.path.abspath(env_p)


@pytest.fixture
Expand Down
26 changes: 17 additions & 9 deletions tests/use-cases/test_3dgan.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
"""
import pytest
import subprocess
import os

CERN_PATH = "use-cases/3dgan"
CKPT_PATH = "3dgan-inference.pth"
CKPT_NAME = "3dgan-inference.pth"


@pytest.mark.skip("deprecated")
Expand All @@ -15,23 +16,25 @@ def test_structure_3dgan(check_folder_structure):


@pytest.mark.functional
def test_3dgan_train(torch_env, install_requirements):
def test_3dgan_train(torch_env, tmp_test_dir, install_requirements):
"""
Test 3DGAN torch lightning trainer by running it end-to-end.
"""
install_requirements(CERN_PATH, torch_env)
conf = os.path.join(os.path.abspath(CERN_PATH), 'pipeline.yaml')
trainer_params = "pipeline.init_args.steps.training_step.init_args"
cmd = (f"{torch_env}/bin/itwinai exec-pipeline "
f"--config pipeline.yaml "
f"--config {conf} "
f'-o {trainer_params}.config.trainer.accelerator=cpu '
f'-o {trainer_params}.config.trainer.strategy=auto '
)
subprocess.run(cmd.split(), check=True, cwd=CERN_PATH)
subprocess.run(cmd.split(), check=True, cwd=tmp_test_dir)


@pytest.mark.functional
def test_3dgan_inference(
torch_env,
tmp_test_dir,
install_requirements,
# fake_model_checkpoint
):
Expand All @@ -41,21 +44,26 @@ def test_3dgan_inference(
install_requirements(CERN_PATH, torch_env)

# Create fake inference dataset and checkpoint
cmd = f"{torch_env}/bin/python create_inference_sample.py"
subprocess.run(cmd.split(), check=True, cwd=CERN_PATH)
exec = os.path.join(os.path.abspath(CERN_PATH),
'create_inference_sample.py')
cmd = (f"{torch_env}/bin/python {exec} "
f"--root {tmp_test_dir} "
f"--ckpt-name {CKPT_NAME}")
subprocess.run(cmd.split(), check=True, cwd=tmp_test_dir)

# Test inference
conf = os.path.join(os.path.abspath(CERN_PATH), 'inference-pipeline.yaml')
getter_params = "pipeline.init_args.steps.dataloading_step.init_args"
trainer_params = "pipeline.init_args.steps.inference_step.init_args"
logger_params = trainer_params + ".config.trainer.logger.init_args"
data_params = trainer_params + ".config.data.init_args"
saver_params = "pipeline.init_args.steps.saver_step.init_args"
cmd = (
f'{torch_env}/bin/itwinai exec-pipeline '
'--config inference-pipeline.yaml '
f'--config {conf} '
f'-o {getter_params}.data_path=exp_data '
f'-o {trainer_params}.model.init_args.model_uri={CKPT_PATH} '
f'-o {trainer_params}.config.trainer.accelerator=cpu '
f'-o {trainer_params}.model.init_args.model_uri={CKPT_NAME} '
f'-o {trainer_params}.config.trainer.accelerator=auto '
f'-o {trainer_params}.config.trainer.strategy=auto '
f'-o {logger_params}.save_dir=ml_logs/mlflow_logs '
f'-o {data_params}.datapath=exp_data/*/*.h5 '
Expand Down
8 changes: 5 additions & 3 deletions tests/use-cases/test_cyclones.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import pytest
import subprocess
import os

CYCLONES_PATH = "use-cases/cyclones"

Expand All @@ -19,12 +20,13 @@ def test_structure_cyclones(check_folder_structure):

@pytest.mark.functional
@pytest.mark.memory_heavy
def test_cyclones_train_tf(tf_env, install_requirements):
def test_cyclones_train_tf(tf_env, tmp_test_dir, install_requirements):
"""
Test Cyclones tensorflow trainer by running it end-to-end.
"""
# TODO: create a small sample dataset for tests only
install_requirements(CYCLONES_PATH, tf_env)
pipe = os.path.join(os.path.abspath(CYCLONES_PATH), 'pipeline.yaml')
cmd = (f"{tf_env}/bin/python train.py "
f"-p pipeline.yaml")
subprocess.run(cmd.split(), check=True, cwd=CYCLONES_PATH)
f"-p {pipe}")
subprocess.run(cmd.split(), check=True, cwd=tmp_test_dir)
42 changes: 27 additions & 15 deletions tests/use-cases/test_mnist.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import pytest
import subprocess
import os
# from itwinai.cli import exec_pipeline

TORCH_PATH = "use-cases/mnist/torch"
Expand All @@ -33,7 +34,7 @@ def test_structure_mnist_tf(check_folder_structure):


@pytest.mark.functional
def test_mnist_train_torch(torch_env, install_requirements):
def test_mnist_train_torch(torch_env, tmp_test_dir, install_requirements):
"""
Test MNIST torch native trainer by running it end-to-end.
Expand All @@ -42,13 +43,14 @@ def test_mnist_train_torch(torch_env, install_requirements):
>>> export TORCH_ENV="my_env"
"""
install_requirements(TORCH_PATH, torch_env)
conf = os.path.join(os.path.abspath(TORCH_PATH), 'config.yaml')
cmd = (f"{torch_env}/bin/itwinai exec-pipeline "
f"--config config.yaml --pipe-key training_pipeline")
subprocess.run(cmd.split(), check=True, cwd=TORCH_PATH)
f"--config {conf} --pipe-key training_pipeline")
subprocess.run(cmd.split(), check=True, cwd=tmp_test_dir)


@pytest.mark.functional
def test_mnist_inference_torch(torch_env, install_requirements):
def test_mnist_inference_torch(torch_env, tmp_test_dir, install_requirements):
"""
Test MNIST torch native inference by running it end-to-end.
Expand All @@ -59,36 +61,46 @@ def test_mnist_inference_torch(torch_env, install_requirements):
install_requirements(TORCH_PATH, torch_env)

# Create fake inference dataset and checkpoint
cmd = f"{torch_env}/bin/python create_inference_sample.py"
subprocess.run(cmd.split(), check=True, cwd=TORCH_PATH)
exec = os.path.join(os.path.abspath(TORCH_PATH),
'create_inference_sample.py')
cmd = (f"{torch_env}/bin/python {exec} "
f"--root {tmp_test_dir}")
subprocess.run(cmd.split(), check=True, cwd=tmp_test_dir)

# Test inference
conf = os.path.join(os.path.abspath(TORCH_PATH), 'config.yaml')
cmd = (f"{torch_env}/bin/itwinai exec-pipeline "
f"--config config.yaml --pipe-key inference_pipeline")
subprocess.run(cmd.split(), check=True, cwd=TORCH_PATH)
f"--config {conf} --pipe-key inference_pipeline")
subprocess.run(cmd.split(), check=True, cwd=tmp_test_dir)


@pytest.mark.functional
def test_mnist_train_torch_lightning(torch_env, install_requirements):
def test_mnist_train_torch_lightning(
torch_env,
tmp_test_dir,
install_requirements
):
"""
Test MNIST torch lightning trainer by running it end-to-end.
To set the torch env path set the ``TORCH_ENV`` env variable:
>>> export TORCH_ENV="my_env"
"""
install_requirements(TORCH_PATH, torch_env)
install_requirements(LIGHTNING_PATH, torch_env)
conf = os.path.join(os.path.abspath(LIGHTNING_PATH), 'config.yaml')
cmd = (f"{torch_env}/bin/itwinai exec-pipeline "
f"--config config.yaml --pipe-key training_pipeline")
subprocess.run(cmd.split(), check=True, cwd=LIGHTNING_PATH)
f"--config {conf} --pipe-key training_pipeline")
subprocess.run(cmd.split(), check=True, cwd=tmp_test_dir)


@pytest.mark.functional
def test_mnist_train_tf(tf_env, install_requirements):
def test_mnist_train_tf(tf_env, tmp_test_dir, install_requirements):
"""
Test MNIST tensorflow trainer by running it end-to-end.
"""
install_requirements(TF_PATH, tf_env)
conf = os.path.join(os.path.abspath(TF_PATH), 'pipeline.yaml')
cmd = (f"{tf_env}/bin/itwinai exec-pipeline "
f"--config pipeline.yaml --pipe-key pipeline")
subprocess.run(cmd.split(), check=True, cwd=TF_PATH)
f"--config {conf} --pipe-key pipeline")
subprocess.run(cmd.split(), check=True, cwd=tmp_test_dir)
Binary file added tfrecords/.DS_Store
Binary file not shown.
21 changes: 17 additions & 4 deletions use-cases/3dgan/create_inference_sample.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,23 @@
"""Create a simple inference dataset sample and a checkpoint."""

import argparse
import os
import torch
from model import ThreeDGAN

CKPT_PATH = "3dgan-inference.pth"

if __name__ == "__main__":
from model import ThreeDGAN
def create_checkpoint(
root: str = '.',
ckpt_name: str = "3dgan-inference.pth"
):
ckpt_path = os.path.join(root, ckpt_name)
net = ThreeDGAN()
torch.save(net, CKPT_PATH)
torch.save(net, ckpt_path)


if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--root", type=str, default='.')
parser.add_argument("--ckpt-name", type=str, default="3dgan-inference.pth")
args = parser.parse_args()
create_checkpoint(vars(args))
3 changes: 2 additions & 1 deletion use-cases/3dgan/dataloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ def execute(self):

gdown.download_folder(
url=self.data_url, quiet=False,
output=self.data_path
output=self.data_path,
verify=False
)


Expand Down
1 change: 1 addition & 0 deletions use-cases/cyclones/dataloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ def setup_config(self, config: Dict) -> None:
if not exists(join(root_dir, self.data_path)):
gdown.download_folder(
url=self.data_url, quiet=False,
verify=False,
output=join(root_dir, self.data_path)
)

Expand Down
10 changes: 9 additions & 1 deletion use-cases/mnist/torch/create_inference_sample.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import torch
import os
import argparse

from model import Net
from dataloader import InferenceMNIST
Expand Down Expand Up @@ -31,4 +32,11 @@ def mnist_torch_inference_files(


if __name__ == "__main__":
mnist_torch_inference_files()
parser = argparse.ArgumentParser()
parser.add_argument("--root", type=str, default='.')
parser.add_argument("--samples-path", type=str,
default='mnist-sample-data')
parser.add_argument("--model-name", type=str,
default='mnist-pre-trained.pth')
args = parser.parse_args()
mnist_torch_inference_files(**vars(args))

0 comments on commit 1ef12e7

Please sign in to comment.