From dee8ba5beb472f811e23bea46b6c48a0c52fe57a Mon Sep 17 00:00:00 2001 From: ChristianOertlin Date: Thu, 16 May 2024 13:33:07 +0200 Subject: [PATCH 01/42] add(validate pacbio transfer cli) (#3226) (patch) # Description Add a validate pacbio transfer cli command. --- cg/cli/base.py | 2 + cg/cli/validate.py | 23 ++++ cg/constants/file_transfer_service.py | 4 + cg/models/cg_config.py | 22 ++++ .../validate_file_transfer_service.py | 70 ++++++++++ .../validate_pacbio_file_transfer_service.py | 97 ++++++++++++++ cg/utils/files.py | 17 +++ tests/conftest.py | 28 ++++ .../pacbio_transfer_done | 7 + .../pacbio_transfer_done_fail | 8 ++ .../conftest.py | 105 +++++++++++++++ .../test_validate_file_transfer_service.py | 120 ++++++++++++++++++ ...t_validate_pacbio_file_transfer_service.py | 88 +++++++++++++ tests/utils/test_files.py | 18 +++ 14 files changed, 609 insertions(+) create mode 100644 cg/cli/validate.py create mode 100644 cg/constants/file_transfer_service.py create mode 100644 cg/services/validate_file_transfer_service/validate_file_transfer_service.py create mode 100644 cg/services/validate_file_transfer_service/validate_pacbio_file_transfer_service.py create mode 100644 tests/fixtures/services/validate_file_transfer_service/pacbio_transfer_done create mode 100644 tests/fixtures/services/validate_file_transfer_service/pacbio_transfer_done_fail create mode 100644 tests/services/validate_file_transfer_service/conftest.py create mode 100644 tests/services/validate_file_transfer_service/test_validate_file_transfer_service.py create mode 100644 tests/services/validate_file_transfer_service/test_validate_pacbio_file_transfer_service.py diff --git a/cg/cli/base.py b/cg/cli/base.py index ebe4d48628..633560995f 100644 --- a/cg/cli/base.py +++ b/cg/cli/base.py @@ -9,6 +9,7 @@ from sqlalchemy.orm import scoped_session import cg +from cg.cli.validate import validate from cg.cli.add import add as add_cmd from cg.cli.archive import archive from cg.cli.backup import backup @@ -115,3 +116,4 @@ def init(context: CGConfig, reset: bool, force: bool): base.add_command(demultiplex_cmd) base.add_command(generate_cmd) base.add_command(downsample) +base.add_command(validate) diff --git a/cg/cli/validate.py b/cg/cli/validate.py new file mode 100644 index 0000000000..45e8882139 --- /dev/null +++ b/cg/cli/validate.py @@ -0,0 +1,23 @@ +import logging + +import click +from cg.models.cg_config import CGConfig +from cg.services.validate_file_transfer_service.validate_pacbio_file_transfer_service import ( + ValidatePacbioFileTransferService, +) + + +LOG = logging.getLogger(__name__) + + +@click.group() +def validate(): + """Validation of processes in cg.""" + + +@validate.command("pacbio-transfer") +@click.pass_obj +def validate_pacbio_transfer(context: CGConfig): + """Validate that the PacBio transfer is correct.""" + validate_service = ValidatePacbioFileTransferService(config=context) + validate_service.validate_all_transfer_done() diff --git a/cg/constants/file_transfer_service.py b/cg/constants/file_transfer_service.py new file mode 100644 index 0000000000..a984a51bbe --- /dev/null +++ b/cg/constants/file_transfer_service.py @@ -0,0 +1,4 @@ +"""Constants for manifest files.""" + +TRANSFER_VALIDATED_FILE = "transfer_validated.txt" +PACBIO_MANIFEST_FILE_PATTERN = "transferdone" diff --git a/cg/models/cg_config.py b/cg/models/cg_config.py index 39bb08873d..dfd6b12b0f 100644 --- a/cg/models/cg_config.py +++ b/cg/models/cg_config.py @@ -301,6 +301,27 @@ class DataFlowConfig(BaseModel): archive_repository: str +class PacbioConfig(BaseModel): + data_dir: str + systemd_trigger_dir: str + + +class OxfordNanoporeConfig(BaseModel): + data_dir: str + systemd_trigger_dir: str + + +class IlluminaConfig(BaseModel): + flow_cell_runs_dir: str + demultiplexed_runs_dir: str + + +class RunInstruments(BaseModel): + pacbio: PacbioConfig + nanopore: OxfordNanoporeConfig + illumina: IlluminaConfig + + class CGConfig(BaseModel): database: str delivery_path: str @@ -314,6 +335,7 @@ class CGConfig(BaseModel): tower_binary_path: str max_flowcells: int | None data_input: DataInput | None = None + run_instruments: RunInstruments # Base APIs that always should exist status_db_: Store | None = None housekeeper: HousekeeperConfig diff --git a/cg/services/validate_file_transfer_service/validate_file_transfer_service.py b/cg/services/validate_file_transfer_service/validate_file_transfer_service.py new file mode 100644 index 0000000000..5fcf4554d2 --- /dev/null +++ b/cg/services/validate_file_transfer_service/validate_file_transfer_service.py @@ -0,0 +1,70 @@ +from pathlib import Path + +from cg.io.controller import ReadFile +from cg.utils.files import get_file_in_directory, get_files_in_directory_with_pattern + + +class ValidateFileTransferService: + """Service to validate file transfers.""" + + @staticmethod + def get_manifest_file_content(manifest_file: Path, manifest_file_format: str) -> list[str]: + """Get the content of the manifest file.""" + file_reader = ReadFile() + return file_reader.get_content_from_file( + file_format=manifest_file_format, file_path=manifest_file + ) + + @staticmethod + def is_valid_path(line: str): + return "/" in line + + def extract_file_names_from_manifest(self, manifest_content: any) -> list[str]: + """ + Extract the file paths from the manifest content. + A file path is expected to contain at least one '/' character. + """ + file_names: list[str] = [] + for line in manifest_content: + if self.is_valid_path(line): + formatted_line: str = line.rstrip() + file_names.append(Path(formatted_line).name) + return file_names + + @staticmethod + def is_file_in_directory_tree(file_name: str, source_dir: Path) -> bool: + """Check if a file is present in the directory tree.""" + try: + if get_file_in_directory(directory=source_dir, file_name=file_name): + return True + except FileNotFoundError: + return False + + @staticmethod + def get_manifest_file_paths(source_dir: Path, pattern: str) -> list[Path]: + return get_files_in_directory_with_pattern(directory=source_dir, pattern=pattern) + + def get_files_in_manifest(self, manifest_file: Path, manifest_file_format: str) -> list[str]: + """Get the files listed in the manifest file.""" + manifest_content: list[str] = self.get_manifest_file_content( + manifest_file=manifest_file, manifest_file_format=manifest_file_format + ) + return self.extract_file_names_from_manifest(manifest_content) + + def are_all_files_present(self, files_to_validate: list[str], source_dir: Path) -> bool: + """Check if all files are present in the directory tree.""" + for file_name in files_to_validate: + if not self.is_file_in_directory_tree(file_name=file_name, source_dir=source_dir): + return False + return True + + def is_transfer_completed( + self, manifest_file: Path, source_dir: Path, manifest_file_format: str + ) -> bool: + """Validate all files listed in the manifest are present in the directory tree.""" + files_in_manifest: list[str] = self.get_files_in_manifest( + manifest_file=manifest_file, manifest_file_format=manifest_file_format + ) + return self.are_all_files_present( + files_to_validate=files_in_manifest, source_dir=source_dir + ) diff --git a/cg/services/validate_file_transfer_service/validate_pacbio_file_transfer_service.py b/cg/services/validate_file_transfer_service/validate_pacbio_file_transfer_service.py new file mode 100644 index 0000000000..67dd77d1dd --- /dev/null +++ b/cg/services/validate_file_transfer_service/validate_pacbio_file_transfer_service.py @@ -0,0 +1,97 @@ +from pathlib import Path +import logging + +from cg.constants.file_transfer_service import PACBIO_MANIFEST_FILE_PATTERN, TRANSFER_VALIDATED_FILE +from cg.constants.constants import FileFormat +from cg.io.controller import WriteFile +from cg.models.cg_config import CGConfig +from cg.services.validate_file_transfer_service.validate_file_transfer_service import ( + ValidateFileTransferService, +) + + +LOG = logging.getLogger(__name__) + + +class ValidatePacbioFileTransferService(ValidateFileTransferService): + + def __init__(self, config: CGConfig): + super().__init__() + self.config: CGConfig = config + self.data_dir: Path = Path(self.config.run_instruments.pacbio.data_dir) + self.trigger_dir: Path = Path(self.config.run_instruments.pacbio.systemd_trigger_dir) + + @staticmethod + def get_run_id(manifest_file_path: Path) -> str: + """ + Get the run name from the path of the manifest file. + Example: r84202_20240307_145215/1_C01/metadata/m84202_240307_145611_s3.transferdone + will return r84202_20240307_145215. + """ + return manifest_file_path.parts[-4] + + def get_smrt_cell_id(self, manifest_file_path: Path) -> str: + """ + Get the run name from the path of the manifest file. + Example: r84202_20240307_145215/1_C01/metadata/m84202_240307_145611_s3.transferdone + will return 1_C01. + """ + return self.get_smrt_cell_path(manifest_file_path).name + + @staticmethod + def get_smrt_cell_path(manifest_file_path: Path) -> Path: + """ + Get the run name from the path of the manifest file. + Example: r84202_20240307_145215/1_C01/metadata/m84202_240307_145611_s3.transferdone + will return r84202_20240307_145215/1_C01. + """ + return Path(manifest_file_path.parts[-3]) + + @staticmethod + def transfer_validated_file_name(manifest_file_path: Path) -> str: + return f"{manifest_file_path.parent}/{TRANSFER_VALIDATED_FILE}" + + def create_validated_transfer_file(self, manifest_file_path: Path) -> None: + file_name: Path = Path(self.transfer_validated_file_name(manifest_file_path)) + writer = WriteFile() + writer.write_file_from_content(file_path=file_name, content="", file_format=FileFormat.TXT) + LOG.debug(f"Created validated transfer file {file_name}") + + def is_transfer_validated(self, manifest_file_path: Path) -> bool: + return Path(self.transfer_validated_file_name(manifest_file_path)).exists() + + def create_systemd_trigger_file(self, manifest_file_path: Path) -> None: + systemd_trigger_file_name: Path = Path( + self.trigger_dir, + f"{self.get_run_id(manifest_file_path)}-{self.get_smrt_cell_id(manifest_file_path)}", + ) + + writer = WriteFile() + writer.write_file_from_content( + file_path=systemd_trigger_file_name, content="", file_format=FileFormat.TXT + ) + LOG.debug(f"Created systemd trigger file {systemd_trigger_file_name}") + + def validate_transfer_done(self, manifest_file_path: Path) -> None: + if not self.is_transfer_completed( + manifest_file=manifest_file_path, + source_dir=self.get_smrt_cell_path(manifest_file_path), + manifest_file_format=FileFormat.TXT, + ): + LOG.info( + f"Transfer not done for run {self.get_run_id(manifest_file_path)} and smrt cell {self.get_smrt_cell_id(manifest_file_path)}" + ) + return + self.create_validated_transfer_file(manifest_file_path) + self.create_systemd_trigger_file(manifest_file_path) + LOG.info( + f"Transfer validated for run {self.get_run_id(manifest_file_path)} and smrt cell {self.get_smrt_cell_id(manifest_file_path)}" + ) + + def validate_all_transfer_done(self) -> None: + manifest_file_paths = self.get_manifest_file_paths( + source_dir=self.data_dir, pattern=PACBIO_MANIFEST_FILE_PATTERN + ) + for manifest_file_path in manifest_file_paths: + if not self.is_transfer_validated(manifest_file_path): + self.validate_transfer_done(manifest_file_path) diff --git a/cg/utils/files.py b/cg/utils/files.py index 5e6e8b5907..838c28ca96 100644 --- a/cg/utils/files.py +++ b/cg/utils/files.py @@ -23,6 +23,23 @@ def get_file_in_directory(directory: Path, file_name: str) -> Path: raise FileNotFoundError(f"File {file_name} not found in {directory}") +def get_files_in_directory_with_pattern(directory: Path, pattern: str) -> list[Path]: + """Get files with a pattern in a directory and subdirectories. + Raises: + FileNotFoundError: If no files with the pattern can be found. + """ + files_with_pattern: list[Path] = [] + if not directory.is_dir() or not directory.exists(): + raise FileNotFoundError(f"Directory {directory} does not exist") + for directory_path, _, files in os.walk(directory): + for file in files: + if pattern in file: + files_with_pattern.append(Path(directory_path, file)) + if not files_with_pattern: + raise FileNotFoundError(f"No files with pattern {pattern} found in {directory}") + return files_with_pattern + + def get_files_matching_pattern(directory: Path, pattern: str) -> list[Path]: """Search for all files in a directory that match a pattern.""" return list(directory.glob(pattern)) diff --git a/tests/conftest.py b/tests/conftest.py index a19af5c31d..0572ff32cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -319,6 +319,20 @@ def base_config_dict() -> dict: "illumina_flow_cells_directory": "path/to/flow_cells", "illumina_demultiplexed_runs_directory": "path/to/demultiplexed_flow_cells_dir", "nanopore_data_directory": "path/to/nanopore_data_directory", + "run_instruments": { + "pacbio": { + "data_dir": "path/to/data_directory", + "systemd_trigger_dir": "path/to/trigger_directory", + }, + "nanopore": { + "data_dir": "path/to/data_directory", + "systemd_trigger_dir": "path/to/ptrigger_directory", + }, + "illumina": { + "flow_cell_runs_dir": "path/to/flow_cells", + "demultiplexed_runs_dir": "path/to/demultiplexed_flow_cells_dir", + }, + }, "downsample": { "downsample_dir": "path/to/downsample_dir", "downsample_script": "downsample.sh", @@ -1783,6 +1797,20 @@ def context_config( "illumina_flow_cells_directory": str(illumina_flow_cells_directory), "illumina_demultiplexed_runs_directory": str(illumina_demultiplexed_runs_directory), "nanopore_data_directory": "path/to/nanopore_data_directory", + "run_instruments": { + "pacbio": { + "data_dir": "path/to/pacbio_data__directory", + "systemd_trigger_dir": "path/to/pacbio_trigger_directory", + }, + "nanopore": { + "data_dir": "path/to/nanopore_data_directory", + "systemd_trigger_dir": "path/to/nanopore_trigger_directory", + }, + "illumina": { + "flow_cell_runs_dir": str(illumina_flow_cells_directory), + "demultiplexed_runs_dir": str(illumina_demultiplexed_runs_directory), + }, + }, "downsample": { "downsample_dir": str(downsample_dir), "downsample_script": "downsample.sh", diff --git a/tests/fixtures/services/validate_file_transfer_service/pacbio_transfer_done b/tests/fixtures/services/validate_file_transfer_service/pacbio_transfer_done new file mode 100644 index 0000000000..64b90d21d4 --- /dev/null +++ b/tests/fixtures/services/validate_file_transfer_service/pacbio_transfer_done @@ -0,0 +1,7 @@ +Transfer Manifest nfiles: 5 +AcqId 6f44afc4-e483-4f0a-acf0-596c229742d6 transfer list size : 5 +/this/path/is/irrelevant/file1 +/this/path/is/irrelevant/file2 +/this/path/is/irrelevant/file3 +/this/path/is/irrelevant/file4 +/this/path/is/irrelevant/file5 \ No newline at end of file diff --git a/tests/fixtures/services/validate_file_transfer_service/pacbio_transfer_done_fail b/tests/fixtures/services/validate_file_transfer_service/pacbio_transfer_done_fail new file mode 100644 index 0000000000..5192d63b74 --- /dev/null +++ b/tests/fixtures/services/validate_file_transfer_service/pacbio_transfer_done_fail @@ -0,0 +1,8 @@ +Transfer Manifest nfiles: 6 +AcqId 6f44afc4-e483-4f0a-acf0-596c229742d6 transfer list size : 6 +/this/path/is/irrelevant/file1 +/this/path/is/irrelevant/file2 +/this/path/is/irrelevant/file3 +/this/path/is/irrelevant/file4 +/this/path/is/irrelevant/file5 +/this/path/is/irrelevant/file6 \ No newline at end of file diff --git a/tests/services/validate_file_transfer_service/conftest.py b/tests/services/validate_file_transfer_service/conftest.py new file mode 100644 index 0000000000..aadfb97583 --- /dev/null +++ b/tests/services/validate_file_transfer_service/conftest.py @@ -0,0 +1,105 @@ +from pathlib import Path + +import pytest + +from cg.models.cg_config import CGConfig +from cg.services.validate_file_transfer_service.validate_file_transfer_service import ( + ValidateFileTransferService, +) +from cg.services.validate_file_transfer_service.validate_pacbio_file_transfer_service import ( + ValidatePacbioFileTransferService, +) + + +@pytest.fixture +def manifest_file(fixtures_dir: Path) -> Path: + """Return the path to a manifest file.""" + return Path(fixtures_dir, "services", "validate_file_transfer_service", "pacbio_transfer_done") + + +@pytest.fixture +def manifest_file_fail(fixtures_dir: Path) -> Path: + """Return the path to a manifest file.""" + return Path( + fixtures_dir, "services", "validate_file_transfer_service", "pacbio_transfer_done_fail" + ) + + +@pytest.fixture +def transfer_source_dir(tmp_path: Path) -> Path: + """Return the path to a source directory with files in different directories.""" + tmp_path.mkdir(exist_ok=True) + tmp_path.joinpath("tree_1").mkdir() + tmp_path.joinpath("tree_1/tree2").mkdir() + tmp_path.joinpath("tree_1/tree2/tree3").mkdir() + tmp_path.joinpath("file1").touch() + tmp_path.joinpath("tree_1/file2").touch() + tmp_path.joinpath("tree_1/file3").touch() + tmp_path.joinpath("tree_1/tree2/file4").touch() + tmp_path.joinpath("tree_1/tree2/tree3/file5").touch() + return tmp_path + + +@pytest.fixture +def validate_file_transfer_service() -> ValidateFileTransferService: + """Return the validate file transfer service.""" + return ValidateFileTransferService() + + +@pytest.fixture +def expected_file_names_in_manifest() -> list[str]: + file_names: list[str] = [] + for i in range(1, 6): + file_names.append(f"file{i}") + return file_names + + +@pytest.fixture +def pacbio_run_id() -> str: + """Return a PacBio run ID.""" + return "r1123_221421_12321" + + +@pytest.fixture +def smrt_cell_id() -> str: + """Return a PacBio smrt cell ID.""" + return "1_A1_0" + + +@pytest.fixture +def metadata_dir_name() -> str: + """Return a metadata directory name.""" + return "metadata" + + +@pytest.fixture +def tmp_manifest_file_name(pacbio_run_id: str, smrt_cell_id: str) -> str: + """Return a temporary manifest file name.""" + return f"{pacbio_run_id}_{smrt_cell_id}.transferdone" + + +@pytest.fixture +def pacbio_runs_dir( + tmp_path: Path, pacbio_run_id: str, smrt_cell_id, metadata_dir_name: str, tmp_manifest_file_name +) -> Path: + """Return the path to a directory with PacBio runs.""" + tmp_path.mkdir(exist_ok=True) + tmp_path.joinpath(pacbio_run_id).mkdir() + tmp_path.joinpath(pacbio_run_id).joinpath(smrt_cell_id).mkdir() + tmp_path.joinpath(pacbio_run_id).joinpath(smrt_cell_id).joinpath(metadata_dir_name).mkdir() + tmp_path.joinpath(pacbio_run_id).joinpath(smrt_cell_id).joinpath(metadata_dir_name).joinpath( + tmp_manifest_file_name + ).touch() + tmp_path.joinpath("not_this_dir").mkdir() + return tmp_path + + +@pytest.fixture +def validate_pacbio_service( + cg_context: CGConfig, pacbio_runs_dir: Path +) -> ValidatePacbioFileTransferService: + """Return a PacBio file transfer service.""" + validate_pacbio_file_transfer_service = ValidatePacbioFileTransferService(config=cg_context) + validate_pacbio_file_transfer_service.data_dir = pacbio_runs_dir.as_posix() + validate_pacbio_file_transfer_service.trigger_dir = pacbio_runs_dir.as_posix() + return validate_pacbio_file_transfer_service diff --git a/tests/services/validate_file_transfer_service/test_validate_file_transfer_service.py b/tests/services/validate_file_transfer_service/test_validate_file_transfer_service.py new file mode 100644 index 0000000000..c99acd754a --- /dev/null +++ b/tests/services/validate_file_transfer_service/test_validate_file_transfer_service.py @@ -0,0 +1,120 @@ +"""Module to test the validate file transfer service.""" + +from pathlib import Path + +from cg.constants.constants import FileFormat +from cg.services.validate_file_transfer_service.validate_file_transfer_service import ( + ValidateFileTransferService, +) + + +def test_get_manifest_content( + validate_file_transfer_service: ValidateFileTransferService, + manifest_file: Path, +): + """Test the get manifest file content method.""" + # GIVEN a manifest file + + # WHEN getting the content of the manifest file + manifest_content: any = validate_file_transfer_service.get_manifest_file_content( + manifest_file=manifest_file, manifest_file_format=FileFormat.TXT + ) + + # THEN assert that the content is a list + assert isinstance(manifest_content, list) + + +def test_get_file_names_from_content( + validate_file_transfer_service: ValidateFileTransferService, + manifest_file: Path, + expected_file_names_in_manifest: list[str], +): + """Test the get file names from content method.""" + # GIVEN a manifest file content + manifest_content: dict = validate_file_transfer_service.get_manifest_file_content( + manifest_file=manifest_file, manifest_file_format=FileFormat.TXT + ) + + # WHEN getting the file names from the content + file_names: list[str] = validate_file_transfer_service.extract_file_names_from_manifest( + manifest_content + ) + + # THEN assert that the file names are a list + assert isinstance(file_names, list) + assert len(file_names) == len(expected_file_names_in_manifest) + for file_name in file_names: + assert file_name in expected_file_names_in_manifest + + +def test_is_file_in_directory( + validate_file_transfer_service: ValidateFileTransferService, + transfer_source_dir: Path, + expected_file_names_in_manifest: list[str], +): + """Test the is file in directory method.""" + # GIVEN a source directory and a list of file names + for file_name in expected_file_names_in_manifest: + # WHEN checking if the file is in the directory + is_file_in_directory: bool = validate_file_transfer_service.is_file_in_directory_tree( + file_name=file_name, source_dir=transfer_source_dir + ) + + # THEN assert that the file is in the directory + assert is_file_in_directory + + +def test_validate_by_manifest_file( + validate_file_transfer_service: ValidateFileTransferService, + manifest_file: Path, + transfer_source_dir: Path, +): + """Test the validate by manifest file method.""" + # GIVEN a manifest file and a source directory + + # WHEN validating the files in the manifest file + is_valid: bool = validate_file_transfer_service.is_transfer_completed( + manifest_file=manifest_file, + source_dir=transfer_source_dir, + manifest_file_format=FileFormat.TXT, + ) + + # THEN assert that the files in the manifest are in the directory + assert is_valid + + +def test_validate_by_manifest_file_fail( + validate_file_transfer_service: ValidateFileTransferService, + manifest_file_fail: Path, + transfer_source_dir: Path, +): + """Test the validate by manifest file method.""" + # GIVEN a manifest file and a source directory + + # WHEN validating the files in the manifest file + is_valid: bool = validate_file_transfer_service.is_transfer_completed( + manifest_file=manifest_file_fail, + source_dir=transfer_source_dir, + manifest_file_format=FileFormat.TXT, + ) + + # THEN assert that the files in the manifest are not in the directory + assert not is_valid + + +def test_get_manifest_file_paths( + expected_file_names_in_manifest: list[str], transfer_source_dir: Path +): + """Test the get manifest file paths method.""" + # GIVEN a source directory + validate_file_transfer_service = ValidateFileTransferService() + + # WHEN getting the using a pattern + manifest_file_paths: list[Path] = validate_file_transfer_service.get_manifest_file_paths( + source_dir=transfer_source_dir, pattern="file" + ) + + # THEN assert that the manifest file paths are a list + assert len(manifest_file_paths) == len(expected_file_names_in_manifest) + for manifest_file_path in manifest_file_paths: + assert manifest_file_path.name in expected_file_names_in_manifest diff --git a/tests/services/validate_file_transfer_service/test_validate_pacbio_file_transfer_service.py b/tests/services/validate_file_transfer_service/test_validate_pacbio_file_transfer_service.py new file mode 100644 index 0000000000..4e010f83f2 --- /dev/null +++ b/tests/services/validate_file_transfer_service/test_validate_pacbio_file_transfer_service.py @@ -0,0 +1,88 @@ +"""Tests for the ValidatePacBioFileTransferService class.""" + +from pathlib import Path + +from cg.constants.file_transfer_service import PACBIO_MANIFEST_FILE_PATTERN, TRANSFER_VALIDATED_FILE +from cg.models.cg_config import CGConfig +from cg.services.validate_file_transfer_service.validate_pacbio_file_transfer_service import ( + ValidatePacbioFileTransferService, +) + + +def test_create_systemd_trigger_file( + pacbio_runs_dir: Path, + pacbio_run_id: str, + validate_pacbio_service: ValidatePacbioFileTransferService, + smrt_cell_id: str, +): + """Test the create systemd trigger file method.""" + # GIVEN a manifest file + + manifest_file: Path = validate_pacbio_service.get_manifest_file_paths( + source_dir=pacbio_runs_dir, pattern=PACBIO_MANIFEST_FILE_PATTERN + )[0] + + # WHEN creating the systemd trigger file + validate_pacbio_service.create_systemd_trigger_file(manifest_file_path=manifest_file) + + # THEN assert that the systemd trigger file was created + assert Path(pacbio_runs_dir, pacbio_run_id + "-" + smrt_cell_id).exists() + + +def test_create_validated_transfer_file( + pacbio_runs_dir: Path, + pacbio_run_id: str, + validate_pacbio_service: ValidatePacbioFileTransferService, +): + """Test the create validated transfer file method.""" + # GIVEN a manifest file + + manifest_file: Path = validate_pacbio_service.get_manifest_file_paths( + source_dir=pacbio_runs_dir, pattern=PACBIO_MANIFEST_FILE_PATTERN + )[0] + + # WHEN creating the validated transfer file + validate_pacbio_service.create_validated_transfer_file(manifest_file_path=manifest_file) + + # THEN assert that the validated transfer file was created + assert Path(manifest_file.parent, TRANSFER_VALIDATED_FILE).exists() + + +def test_get_run_id( + pacbio_runs_dir: Path, + pacbio_run_id: str, + validate_pacbio_service: ValidatePacbioFileTransferService, +): + """Test the get run ID method.""" + # GIVEN a manifest file + + manifest_file: Path = validate_pacbio_service.get_manifest_file_paths( + source_dir=pacbio_runs_dir, pattern=PACBIO_MANIFEST_FILE_PATTERN + )[0] + + # WHEN getting the run ID + extracted_run_id: str = validate_pacbio_service.get_run_id(manifest_file_path=manifest_file) + + # THEN it is the expected run id + assert extracted_run_id == pacbio_run_id + + +def test_get_smrt_cell_id( + pacbio_runs_dir: Path, + smrt_cell_id: str, + validate_pacbio_service: ValidatePacbioFileTransferService, +): + """Test the get smrt cell ID method.""" + # GIVEN a manifest file + + manifest_file: Path = validate_pacbio_service.get_manifest_file_paths( + source_dir=pacbio_runs_dir, pattern=PACBIO_MANIFEST_FILE_PATTERN + )[0] + + # WHEN getting the smrt cell ID + extracted_smrt_cell_id: str = validate_pacbio_service.get_smrt_cell_id( + manifest_file_path=manifest_file + ) + + # THEN it is the expected smrt cell id + assert extracted_smrt_cell_id == smrt_cell_id diff --git a/tests/utils/test_files.py b/tests/utils/test_files.py index 3a0ae28681..419c796212 100644 --- a/tests/utils/test_files.py +++ b/tests/utils/test_files.py @@ -9,18 +9,36 @@ get_file_in_directory, remove_directory_and_contents, rename_file, + get_files_in_directory_with_pattern, ) def test_get_file_in_directory(nested_directory_with_file: Path, some_file: str): """Test function to get a file in a directory and subdirectories.""" # GIVEN a directory with subdirectories with a file + # WHEN getting the file file_path: Path = get_file_in_directory(nested_directory_with_file, some_file) + # THEN assert that the file is returned assert file_path.exists() +def test_get_files_in_directory_by_pattern(nested_directory_with_file: Path, some_file: str): + """Test function to get files with a pattern in a directory and subdirectories.""" + # GIVEN a directory with subdirectories with a file + + # WHEN getting the file + file_paths: list[Path] = get_files_in_directory_with_pattern( + directory=nested_directory_with_file, pattern=some_file + ) + + # THEN assert that the file is returned + for file_path in file_paths: + assert file_path.exists() + assert some_file in file_path.as_posix() + + def test_rename_file(tmp_path: Path): # GIVEN a file path and a renamed file path file_path: Path = Path(tmp_path, "dummy_path") From 776d338df47c6dac217708e5efb0a0c6a1bfe917 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 16 May 2024 11:33:33 +0000 Subject: [PATCH 02/42] =?UTF-8?q?Bump=20version:=2060.7.18=20=E2=86=92=206?= =?UTF-8?q?0.7.19=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f9ecbf3413..2a27386c95 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.18 +current_version = 60.7.19 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 557b1b9c0c..5030f79a8b 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.18" +__version__ = "60.7.19" diff --git a/pyproject.toml b/pyproject.toml index c2dc083bc5..d5f94f34b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.18" +version = "60.7.19" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 4c7b6325a794a2fb341fbf24ce2f569adb773991 Mon Sep 17 00:00:00 2001 From: Vadym Date: Thu, 16 May 2024 14:55:07 +0200 Subject: [PATCH 03/42] Add RIN thresholds for Rnafusion & Tomte delivery reports (#3203) ### Added: - RIN max and min thresholds for delivery report validation. ### Fixed: - Zero integers treated as N/A when parsing metrics for the delivery report. --- cg/constants/report.py | 3 + cg/models/report/metadata.py | 76 ++++++++++++++----------- cg/models/report/validators.py | 20 +++++-- tests/meta/report/test_rnafusion_api.py | 50 +++++++++++++++- tests/models/report/test_validators.py | 50 +++++++++------- 5 files changed, 138 insertions(+), 61 deletions(-) diff --git a/cg/constants/report.py b/cg/constants/report.py index 00820cb492..42c86216c1 100644 --- a/cg/constants/report.py +++ b/cg/constants/report.py @@ -41,6 +41,9 @@ NO_FIELD: str = "Nej" PRECISION: int = 2 +RIN_MAX_THRESHOLD: int = 10 +RIN_MIN_THRESHOLD: int = 1 + REPORT_GENDER: dict[str, str] = { "unknown": "Okänd", "female": "Kvinna", diff --git a/cg/models/report/metadata.py b/cg/models/report/metadata.py index 5f3f0eab0e..c09aaebf52 100644 --- a/cg/models/report/metadata.py +++ b/cg/models/report/metadata.py @@ -1,12 +1,12 @@ -from pydantic import BaseModel, BeforeValidator +from pydantic import BaseModel, BeforeValidator, field_validator from typing_extensions import Annotated -from cg.constants import NA_FIELD +from cg.constants import NA_FIELD, RIN_MIN_THRESHOLD, RIN_MAX_THRESHOLD from cg.models.report.validators import ( get_float_as_percentage, - get_float_as_string, get_gender_as_string, get_report_string, + get_number_as_string, ) @@ -19,8 +19,8 @@ class SampleMetadataModel(BaseModel): duplicates: fraction of mapped sequence that is marked as duplicate; source: workflow """ - million_read_pairs: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - duplicates: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + million_read_pairs: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + duplicates: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD class MipDNASampleMetadataModel(SampleMetadataModel): @@ -36,9 +36,9 @@ class MipDNASampleMetadataModel(SampleMetadataModel): bait_set: Annotated[str, BeforeValidator(get_report_string)] = NA_FIELD gender: Annotated[str, BeforeValidator(get_gender_as_string)] = NA_FIELD - mapped_reads: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - mean_target_coverage: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - pct_10x: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + mapped_reads: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + mean_target_coverage: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + pct_10x: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD class BalsamicSampleMetadataModel(SampleMetadataModel): @@ -49,8 +49,8 @@ class BalsamicSampleMetadataModel(SampleMetadataModel): fold_80: fold 80 base penalty; source: workflow """ - mean_insert_size: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - fold_80: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + mean_insert_size: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + fold_80: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD class BalsamicTargetedSampleMetadataModel(BalsamicSampleMetadataModel): @@ -66,10 +66,10 @@ class BalsamicTargetedSampleMetadataModel(BalsamicSampleMetadataModel): bait_set: Annotated[str, BeforeValidator(get_report_string)] = NA_FIELD bait_set_version: Annotated[str, BeforeValidator(get_report_string)] = NA_FIELD - median_target_coverage: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - pct_250x: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - pct_500x: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - gc_dropout: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + median_target_coverage: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + pct_250x: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + pct_500x: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + gc_dropout: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD class BalsamicWGSSampleMetadataModel(BalsamicSampleMetadataModel): @@ -81,10 +81,10 @@ class BalsamicWGSSampleMetadataModel(BalsamicSampleMetadataModel): pct_60x: fraction of bases that attained at least 15X sequence coverage; source: workflow """ - median_coverage: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - pct_15x: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - pct_60x: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - pct_reads_improper_pairs: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + median_coverage: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + pct_15x: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + pct_60x: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + pct_reads_improper_pairs: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD class SequencingSampleMetadataModel(SampleMetadataModel): @@ -96,7 +96,7 @@ class SequencingSampleMetadataModel(SampleMetadataModel): """ gc_content: Annotated[str, BeforeValidator(get_float_as_percentage)] = NA_FIELD - mean_length_r1: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + mean_length_r1: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD class WTSSampleMetadataModel(SequencingSampleMetadataModel): @@ -115,16 +115,24 @@ class WTSSampleMetadataModel(SequencingSampleMetadataModel): uniquely_mapped_reads: percentage of mapped reads; source: workflow """ - bias_5_3: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - input_amount: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - mrna_bases: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - pct_adapter: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - pct_surviving: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + bias_5_3: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + input_amount: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + mrna_bases: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + pct_adapter: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + pct_surviving: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD q20_rate: Annotated[str, BeforeValidator(get_float_as_percentage)] = NA_FIELD q30_rate: Annotated[str, BeforeValidator(get_float_as_percentage)] = NA_FIELD ribosomal_bases: Annotated[str, BeforeValidator(get_float_as_percentage)] = NA_FIELD - rin: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - uniquely_mapped_reads: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + rin: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + uniquely_mapped_reads: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + + @field_validator("rin") + def ensure_rin_thresholds(cls, rin: str) -> str: + if rin != NA_FIELD: + rin_number = float(rin) + if RIN_MIN_THRESHOLD <= rin_number <= RIN_MAX_THRESHOLD: + return str(rin_number) + return NA_FIELD class RnafusionSampleMetadataModel(WTSSampleMetadataModel): @@ -136,8 +144,8 @@ class RnafusionSampleMetadataModel(WTSSampleMetadataModel): mapped_reads: percentage of reads aligned to the reference sequence; source: workflow """ - insert_size: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - insert_size_peak: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + insert_size: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + insert_size_peak: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD mapped_reads: Annotated[str, BeforeValidator(get_float_as_percentage)] = NA_FIELD @@ -151,10 +159,10 @@ class TaxprofilerSampleMetadataModel(SequencingSampleMetadataModel): million_read_pairs_after_filtering: number of reads after filtering; source: workflow """ - average_read_length: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - mapped_reads: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - mean_length_r2: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - million_read_pairs_after_filtering: Annotated[str, BeforeValidator(get_float_as_string)] = ( + average_read_length: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + mapped_reads: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + mean_length_r2: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + million_read_pairs_after_filtering: Annotated[str, BeforeValidator(get_number_as_string)] = ( NA_FIELD ) @@ -167,5 +175,5 @@ class TomteSampleMetadataModel(WTSSampleMetadataModel): pct_intronic_bases: proportion of genomic bases within intronic regions; source: workflow """ - pct_intergenic_bases: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD - pct_intronic_bases: Annotated[str, BeforeValidator(get_float_as_string)] = NA_FIELD + pct_intergenic_bases: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD + pct_intronic_bases: Annotated[str, BeforeValidator(get_number_as_string)] = NA_FIELD diff --git a/cg/models/report/validators.py b/cg/models/report/validators.py index d6ca2e2208..350e04f049 100644 --- a/cg/models/report/validators.py +++ b/cg/models/report/validators.py @@ -33,14 +33,26 @@ def get_boolean_as_string(value: bool | None) -> str: return NA_FIELD -def get_float_as_string(value: float | None) -> str: - """Return string representation of a float value.""" - return str(round(float(value), PRECISION)) if value or isinstance(value, float) else NA_FIELD +def get_number_as_string(value: Any) -> str: + """ + Return string representation of a number. If None is provided, then it returns N/A. + + Raises: + ValueError: If the input value cannot be converted to a float. + """ + try: + result: str = str(round(float(value), PRECISION)) + return result + except TypeError: + return NA_FIELD + except ValueError: + LOG.error(f"Value {value} cannot be converted to float") + raise def get_float_as_percentage(value: float | None) -> str: """Return string percentage representation of a float value.""" - return get_float_as_string(value * 100) if value or isinstance(value, float) else NA_FIELD + return get_number_as_string(value * 100) if value or isinstance(value, float) else NA_FIELD def get_date_as_string(date: datetime | None) -> str: diff --git a/tests/meta/report/test_rnafusion_api.py b/tests/meta/report/test_rnafusion_api.py index 4d26bb93b6..bb47028d0d 100644 --- a/tests/meta/report/test_rnafusion_api.py +++ b/tests/meta/report/test_rnafusion_api.py @@ -1,9 +1,14 @@ """Test module for the Rnafusion delivery report API.""" +import pytest +from pytest_mock import MockFixture + +from cg.constants import NA_FIELD, RIN_MAX_THRESHOLD, RIN_MIN_THRESHOLD from cg.meta.report.rnafusion import RnafusionReportAPI from cg.models.analysis import NextflowAnalysis from cg.models.report.metadata import RnafusionSampleMetadataModel from cg.store.models import Case, Sample +from tests.mocks.limsmock import MockLimsAPI def test_get_sample_metadata( @@ -11,7 +16,7 @@ def test_get_sample_metadata( sample_id: str, rnafusion_case_id: str, rnafusion_validated_metrics: dict[str, str], - rnafusion_mock_analysis_finish, + rnafusion_mock_analysis_finish: None, ): """Test Rnafusion sample metadata extraction.""" @@ -33,3 +38,46 @@ def test_get_sample_metadata( # THEN the sample metadata should be correctly retrieved and match the expected validated metrics assert sample_metadata.model_dump() == rnafusion_validated_metrics + + +@pytest.mark.parametrize( + "input_rin, expected_rin", + [ + (RIN_MAX_THRESHOLD, str(float(RIN_MAX_THRESHOLD))), # Test for a valid integer input + (RIN_MAX_THRESHOLD + 1, NA_FIELD), # Test for an integer above the allowed threshold + (RIN_MIN_THRESHOLD - 1, NA_FIELD), # Test for an integer below the allowed threshold + (None, NA_FIELD), # Test for a None input + ], +) +def test_ensure_rin_thresholds( + rnafusion_case_id: str, + sample_id: str, + input_rin: int | float, + expected_rin: str, + report_api_rnafusion: RnafusionReportAPI, + rnafusion_mock_analysis_finish: None, + mocker: MockFixture, +): + """Test Rnafusion RIN value validation.""" + + # GIVEN a Rnafusion case and associated sample + case: Case = report_api_rnafusion.status_db.get_case_by_internal_id( + internal_id=rnafusion_case_id + ) + sample: Sample = report_api_rnafusion.status_db.get_sample_by_internal_id(internal_id=sample_id) + + # GIVEN an analysis metadata object + latest_metadata: NextflowAnalysis = report_api_rnafusion.analysis_api.get_latest_metadata( + case_id=rnafusion_case_id + ) + + # GIVEN a specific RIN value + mocker.patch.object(MockLimsAPI, "get_sample_rin", return_value=input_rin) + + # WHEN getting the sample metadata + sample_metadata: RnafusionSampleMetadataModel = report_api_rnafusion.get_sample_metadata( + case=case, sample=sample, analysis_metadata=latest_metadata + ) + + # THEN the sample RIN value should match the expected RIN value + assert sample_metadata.rin == expected_rin diff --git a/tests/models/report/test_validators.py b/tests/models/report/test_validators.py index 82492b8b9c..6d86464f1c 100644 --- a/tests/models/report/test_validators.py +++ b/tests/models/report/test_validators.py @@ -18,13 +18,13 @@ get_boolean_as_string, get_date_as_string, get_delivered_files_as_file_names, - get_float_as_percentage, - get_float_as_string, get_gender_as_string, get_list_as_string, get_path_as_string, get_prep_category_as_string, get_report_string, + get_number_as_string, + get_float_as_percentage, ) @@ -63,30 +63,36 @@ def test_get_boolean_as_string(): assert validated_not_bool_field == NA_FIELD -def test_get_float_as_string(): - """Test the validation of a float value.""" - - # GIVEN a valid float input - float_value: float = 12.3456789 - - # WHEN performing the validation - validated_float_value: str = get_float_as_string(float_value) - - # THEN check if the input value was formatted correctly - assert validated_float_value == "12.35" +@pytest.mark.parametrize( + "input_value, expected_output", + [ + (12.3456789, "12.35"), # Test for a valid float input + (0.0, "0.0"), # Test for float zero input + (5, "5.0"), # Test for a valid integer input + (0, "0.0"), # Test for integer zero input + (None, NA_FIELD), # Test for None input + ("1.2", "1.2"), # Test for valid string input + ("invalid", ValueError), # Test for an invalid string input + ], +) +def test_get_number_as_string(input_value: Any, expected_output: str, caplog: LogCaptureFixture): + """Test the validation and formatting of numbers.""" + # GIVEN a list of number inputs and their expected values -def test_get_float_as_string_zero_input(): - """Tests the validation of a float value when input is zero.""" + if expected_output == ValueError: + # WHEN getting a string representation of a number + with pytest.raises(ValueError): + get_number_as_string(input_value) - # GIVEN a valid float input - float_value: float = 0.0 + # THEN a ValueError should have been raised for an invalid number input + assert f"Value {input_value} cannot be converted to float" in caplog.text + else: + # WHEN getting a string representation of a number + validated_float_value = get_number_as_string(input_value) - # WHEN performing the validation - validated_float_value: str = get_float_as_string(float_value) - - # THEN check if the input value was formatted correctly - assert validated_float_value == "0.0" + # THEN the expected output should be correctly formatted + assert validated_float_value == expected_output def test_get_float_as_percentage(): From c7ddbe50d57ec7f0c32c37addaed44665ffcf486 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 16 May 2024 12:55:35 +0000 Subject: [PATCH 04/42] =?UTF-8?q?Bump=20version:=2060.7.19=20=E2=86=92=206?= =?UTF-8?q?0.7.20=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2a27386c95..bf0d3813ae 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.19 +current_version = 60.7.20 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 5030f79a8b..dd7123ca75 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.19" +__version__ = "60.7.20" diff --git a/pyproject.toml b/pyproject.toml index d5f94f34b4..94835003fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.19" +version = "60.7.20" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From a480578f2b6d9f2d8a0d1a88c4398f079a898ed8 Mon Sep 17 00:00:00 2001 From: Sebastian Allard Date: Thu, 16 May 2024 15:05:57 +0200 Subject: [PATCH 05/42] Use beefier test runner (#3233)(patch) --- .github/workflows/tests_and_coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_and_coverage.yml b/.github/workflows/tests_and_coverage.yml index 0c2a803389..cf9e77110f 100644 --- a/.github/workflows/tests_and_coverage.yml +++ b/.github/workflows/tests_and_coverage.yml @@ -8,7 +8,7 @@ on: jobs: tests-coverage: - runs-on: ubuntu-latest + runs-on: Beefy_Linux steps: - name: Checkout Repository @@ -37,7 +37,7 @@ jobs: - name: Test with pytest & Coveralls run: | - pytest -n auto --cov=cg/ + pytest -n logical --cov=cg/ coveralls env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} From d7a9eb2db7931b611f010120cec6302c8babdb1d Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 16 May 2024 13:06:26 +0000 Subject: [PATCH 06/42] =?UTF-8?q?Bump=20version:=2060.7.20=20=E2=86=92=206?= =?UTF-8?q?0.7.21=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index bf0d3813ae..b5b4ebdfc9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.20 +current_version = 60.7.21 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index dd7123ca75..4fb5caf9e3 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.20" +__version__ = "60.7.21" diff --git a/pyproject.toml b/pyproject.toml index 94835003fd..3535515432 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.20" +version = "60.7.21" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 8a7e9fee9c011a09da2b57c573e821b66c22d2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isak=20Ohlsson=20=C3=85ngnell?= <40887124+islean@users.noreply.github.com> Date: Thu, 16 May 2024 15:15:59 +0200 Subject: [PATCH 07/42] Validate CLI case name (#3237) (patch) ### Fixed - Case name in cg add case is only allowed to contain letters, digits and dashes. --- cg/cli/add.py | 7 ++++++- cg/cli/utils.py | 8 ++++++++ tests/cli/add/test_cli_add_family.py | 4 ++-- tests/utils/test_utils.py | 13 +++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/cg/cli/add.py b/cg/cli/add.py index b3b3e8b1c2..2ff3c5db9b 100644 --- a/cg/cli/add.py +++ b/cg/cli/add.py @@ -2,9 +2,10 @@ import click +from cg.cli.utils import is_case_name_allowed from cg.constants import DataDelivery, Priority, Workflow from cg.constants.archiving import PDC_ARCHIVE_LOCATION -from cg.constants.constants import StatusOptions, DRY_RUN +from cg.constants.constants import DRY_RUN, StatusOptions from cg.constants.subject import Sex from cg.meta.transfer.external_data import ExternalDataAPI from cg.models.cg_config import CGConfig @@ -271,6 +272,10 @@ def add_case( LOG.error(f"{panel_abbreviation}: panel not found") raise click.Abort + if not is_case_name_allowed(name): + LOG.error(f"Case name {name} is only allowed to contain letters, digits and dashes.") + raise click.Abort + new_case: Case = status_db.add_case( data_analysis=data_analysis, data_delivery=data_delivery, diff --git a/cg/cli/utils.py b/cg/cli/utils.py index 3a04f5a714..7a13b0f5f4 100644 --- a/cg/cli/utils.py +++ b/cg/cli/utils.py @@ -1,6 +1,14 @@ +import re + import click def echo_lines(lines: list[str]) -> None: for line in lines: click.echo(line) + + +def is_case_name_allowed(name: str) -> bool: + """Returns true if the given name consists only of letters, numbers and dashes.""" + allowed_pattern: re.Pattern = re.compile("^[A-Za-z0-9-]+$") + return bool(allowed_pattern.fullmatch(name)) diff --git a/tests/cli/add/test_cli_add_family.py b/tests/cli/add/test_cli_add_family.py index 96d98a4497..f90948902a 100644 --- a/tests/cli/add/test_cli_add_family.py +++ b/tests/cli/add/test_cli_add_family.py @@ -24,7 +24,7 @@ def test_add_case_required( customer_id = customer.internal_id panel: Panel = helpers.ensure_panel(store=disk_store) panel_id = panel.name - name = "case_name" + name = "case-name" # WHEN adding a panel result = cli_runner.invoke( @@ -209,7 +209,7 @@ def test_add_case_priority( panel: Panel = helpers.ensure_panel(store=disk_store) panel_id = panel.name - name = "case_name" + name = "case-name" priority = "priority" result = cli_runner.invoke( diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index ea8d8e010d..f0b30de461 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -2,6 +2,7 @@ import pytest +from cg.cli.utils import is_case_name_allowed from cg.utils.utils import get_hamming_distance, get_string_from_list_by_pattern @@ -64,3 +65,15 @@ def test_get_hamming_distance_different_lengths(): str(exc_info.value) == "The two strings must have the same length to calculate distance!" ) + + +@pytest.mark.parametrize( + "case_name, expected_behaviour", + [ + ("valid-case-name123", True), + ("invalid_case_name", False), + ("invalid-special-character()", False), + ], +) +def test_is_case_name_valid(case_name: str, expected_behaviour: bool): + assert is_case_name_allowed(case_name) == expected_behaviour From 3714ad4a0a622ddadd0847f3a73be8dada315845 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 16 May 2024 13:16:27 +0000 Subject: [PATCH 08/42] =?UTF-8?q?Bump=20version:=2060.7.21=20=E2=86=92=206?= =?UTF-8?q?0.7.22=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b5b4ebdfc9..7239b31c14 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.21 +current_version = 60.7.22 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 4fb5caf9e3..b401e79a35 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.21" +__version__ = "60.7.22" diff --git a/pyproject.toml b/pyproject.toml index 3535515432..e704b81258 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.21" +version = "60.7.22" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 3d4cebdbe748cb11229bc8cba4ecb6dc0080bab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isak=20Ohlsson=20=C3=85ngnell?= <40887124+islean@users.noreply.github.com> Date: Fri, 17 May 2024 08:57:42 +0200 Subject: [PATCH 09/42] Fix typing (#3225) (patch) ### Fixed - Path is converted to list of paths in the Microsalt cleaning. --- cg/meta/workflow/microsalt/microsalt.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cg/meta/workflow/microsalt/microsalt.py b/cg/meta/workflow/microsalt/microsalt.py index 06777bbd73..63ca68b954 100644 --- a/cg/meta/workflow/microsalt/microsalt.py +++ b/cg/meta/workflow/microsalt/microsalt.py @@ -17,7 +17,9 @@ from cg.meta.workflow.microsalt.quality_controller.models import QualityResult from cg.meta.workflow.microsalt.utils import get_most_recent_project_directory from cg.models.cg_config import CGConfig -from cg.services.quality_controller.quality_controller_service import QualityControllerService +from cg.services.quality_controller.quality_controller_service import ( + QualityControllerService, +) from cg.store.models import Case, Sample from cg.utils import Process @@ -60,7 +62,8 @@ def clean_run_dir(self, case_id: str, yes: bool, case_path: list[Path] | Path) - ) self.clean_analyses(case_id=case_id) return EXIT_SUCCESS - + if isinstance(case_path, Path): + case_path: list[Path] = [case_path] for analysis_path in case_path: if yes or click.confirm( f"Are you sure you want to remove all files in {analysis_path}?" From fa1736847a1d99c776a78934ac99bf20b532e61b Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Fri, 17 May 2024 06:58:08 +0000 Subject: [PATCH 10/42] =?UTF-8?q?Bump=20version:=2060.7.22=20=E2=86=92=206?= =?UTF-8?q?0.7.23=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7239b31c14..f160dc6594 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.22 +current_version = 60.7.23 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index b401e79a35..5658ed841d 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.22" +__version__ = "60.7.23" diff --git a/pyproject.toml b/pyproject.toml index e704b81258..05d890edab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.22" +version = "60.7.23" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 42fcd35384a2b75142d2ecb0a6a50139d467b91d Mon Sep 17 00:00:00 2001 From: ChristianOertlin Date: Fri, 17 May 2024 11:58:11 +0200 Subject: [PATCH 11/42] add(illumina run metrics table) (#3235) (patch) # Description add new illumina run metrics model --- ...6_6e6c36d5157b_add_illumina_run_metrics.py | 52 +++++++++++++++++++ cg/store/models.py | 31 +++++++++++ 2 files changed, 83 insertions(+) create mode 100644 alembic/versions/2024_05_16_6e6c36d5157b_add_illumina_run_metrics.py diff --git a/alembic/versions/2024_05_16_6e6c36d5157b_add_illumina_run_metrics.py b/alembic/versions/2024_05_16_6e6c36d5157b_add_illumina_run_metrics.py new file mode 100644 index 0000000000..7d4d6dcc4b --- /dev/null +++ b/alembic/versions/2024_05_16_6e6c36d5157b_add_illumina_run_metrics.py @@ -0,0 +1,52 @@ +"""add_illumina_run_metrics + +Revision ID: 6e6c36d5157b +Revises: 5c6de08c4aca +Create Date: 2024-05-16 13:40:36.754552 + +""" + +from enum import StrEnum + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "6e6c36d5157b" +down_revision = "5c6de08c4aca" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "illumina_sequencing_metrics", + sa.Column( + "id", sa.Integer(), sa.ForeignKey("run_metrics.id"), nullable=False, primary_key=True + ), + sa.Column("sequencer_type", sa.String(length=32), nullable=True), + sa.Column("sequencer_name", sa.String(length=32), nullable=True), + sa.Column("sequenced_at", sa.DateTime(), nullable=True), + sa.Column("data_availability", sa.String(length=32), nullable=True), + sa.Column("archived_at", sa.DateTime(), nullable=True), + sa.Column("has_backup", sa.Boolean(), nullable=False), + sa.Column("total_reads", sa.BigInteger(), nullable=True), + sa.Column("total_undetermined_reads", sa.BigInteger(), nullable=True), + sa.Column("percent_q30", sa.Numeric(precision=6, scale=2), nullable=True), + sa.Column("mean_quality_score", sa.Numeric(precision=6, scale=2), nullable=True), + sa.Column("total_yield", sa.BigInteger(), nullable=True), + sa.Column("yield_q30", sa.Numeric(precision=6, scale=2), nullable=True), + sa.Column("cycles", sa.Integer(), nullable=True), + sa.Column("demultiplexing_software", sa.String(length=32), nullable=True), + sa.Column("demultiplexing_software_version", sa.String(length=32), nullable=True), + sa.Column("sequencing_started_at", sa.DateTime(), nullable=True), + sa.Column("sequencing_completed_at", sa.DateTime(), nullable=True), + sa.Column("demultiplexing_started_at", sa.DateTime(), nullable=True), + sa.Column("demultiplexing_completed_at", sa.DateTime(), nullable=True), + ) + + +def downgrade(): + + op.drop_table("illumina_run_metrics") diff --git a/cg/store/models.py b/cg/store/models.py index 35b88590fb..60b1503847 100644 --- a/cg/store/models.py +++ b/cg/store/models.py @@ -989,6 +989,37 @@ class RunMetrics(Base): } +class IlluminaRunMetrics(RunMetrics): + __tablename__ = "illumina_sequencing_metrics" + + id: Mapped[int] = mapped_column(ForeignKey("run_metrics.id"), primary_key=True) + sequencer_type: Mapped[str | None] = mapped_column( + types.Enum("hiseqga", "hiseqx", "novaseq", "novaseqx") + ) + sequencer_name: Mapped[Str32 | None] + sequenced_at: Mapped[datetime | None] + data_availability: Mapped[str | None] = mapped_column( + types.Enum(*(status.value for status in FlowCellStatus)), default="ondisk" + ) + archived_at: Mapped[datetime | None] + has_backup: Mapped[bool] = mapped_column(default=False) + total_reads: Mapped[BigInt | None] + total_undetermined_reads: Mapped[BigInt | None] + percent_q30: Mapped[Num_6_2 | None] + mean_quality_score: Mapped[Num_6_2 | None] + total_yield: Mapped[BigInt | None] + yield_q30: Mapped[Num_6_2 | None] + cycles: Mapped[int | None] + demultiplexing_software: Mapped[Str32 | None] + demultiplexing_software_version: Mapped[Str32 | None] + sequencing_started_at: Mapped[datetime | None] + sequencing_completed_at: Mapped[datetime | None] + demultiplexing_started_at: Mapped[datetime | None] + demultiplexing_completed_at: Mapped[datetime | None] + + __mapper_args__ = {"polymorphic_identity": DeviceType.ILLUMINA} + + class SampleRunMetrics(Base): __tablename__ = "sample_run_metrics" id: Mapped[PrimaryKeyInt] From b51184467b8417199c7d1017325026bcbe3f8dc3 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Fri, 17 May 2024 09:58:37 +0000 Subject: [PATCH 12/42] =?UTF-8?q?Bump=20version:=2060.7.23=20=E2=86=92=206?= =?UTF-8?q?0.7.24=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f160dc6594..7674d335b9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.23 +current_version = 60.7.24 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 5658ed841d..67d06cd387 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.23" +__version__ = "60.7.24" diff --git a/pyproject.toml b/pyproject.toml index 05d890edab..c1b6579628 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.23" +version = "60.7.24" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From bb01f241a4181556e6a4f9f1b1a9dc5be6cd8797 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Mon, 20 May 2024 09:42:49 +0200 Subject: [PATCH 13/42] add IlluminaFlowCellDevice model (#3239) ## Description Closes https://github.com/Clinical-Genomics/add-new-tech/issues/2 ### Added - IlluminaFlowCellDevice model - Alembic migration --- ..._17_fe23de4ed528_add_illumina_flow_cell.py | 31 +++++++++++++++++++ cg/constants/demultiplexing.py | 2 +- cg/store/models.py | 13 ++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/2024_05_17_fe23de4ed528_add_illumina_flow_cell.py diff --git a/alembic/versions/2024_05_17_fe23de4ed528_add_illumina_flow_cell.py b/alembic/versions/2024_05_17_fe23de4ed528_add_illumina_flow_cell.py new file mode 100644 index 0000000000..6f73ca6754 --- /dev/null +++ b/alembic/versions/2024_05_17_fe23de4ed528_add_illumina_flow_cell.py @@ -0,0 +1,31 @@ +"""add_illumina_flow_cell + +Revision ID: fe23de4ed528 +Revises: 6e6c36d5157b +Create Date: 2024-05-17 15:09:12.088324 + +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "fe23de4ed528" +down_revision = "6e6c36d5157b" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "illumina_flow_cell", + sa.Column( + "id", sa.Integer(), sa.ForeignKey("run_device.id"), nullable=False, primary_key=True + ), + sa.Column("model", sa.String(length=32), nullable=False), + ) + + +def downgrade(): + op.drop_table("illumina_flow_cell") diff --git a/cg/constants/demultiplexing.py b/cg/constants/demultiplexing.py index 01a12d0996..aec437e6d0 100644 --- a/cg/constants/demultiplexing.py +++ b/cg/constants/demultiplexing.py @@ -1,6 +1,6 @@ """Constants related to demultiplexing.""" -from enum import StrEnum +from enum import Enum, StrEnum from pydantic import BaseModel diff --git a/cg/store/models.py b/cg/store/models.py index 60b1503847..117f12115c 100644 --- a/cg/store/models.py +++ b/cg/store/models.py @@ -975,6 +975,19 @@ class RunDevice(Base): } +class IlluminaFlowCell(RunDevice): + """Model for storing Illumina flow cells.""" + + __tablename__ = "illumina_flow_cell" + + id: Mapped[int] = mapped_column(ForeignKey("run_device.id"), primary_key=True) + model: Mapped[str | None] = mapped_column( + types.Enum("10B", "25B", "1.5B", "S1", "S2", "S4", "SP") + ) + + __mapper_args__ = {"polymorphic_identity": DeviceType.ILLUMINA} + + class RunMetrics(Base): """Model for storing run devices.""" From b624356e714007cb99b4f9d76ee9642a5a5fc6ac Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Mon, 20 May 2024 07:43:16 +0000 Subject: [PATCH 14/42] =?UTF-8?q?Bump=20version:=2060.7.24=20=E2=86=92=206?= =?UTF-8?q?0.7.25=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7674d335b9..5ba08a7671 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.24 +current_version = 60.7.25 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 67d06cd387..8c847c8248 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.24" +__version__ = "60.7.25" diff --git a/pyproject.toml b/pyproject.toml index c1b6579628..8074259662 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.24" +version = "60.7.25" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 445a1d4a98dfa791df1fc8a91f81349e8ee582de Mon Sep 17 00:00:00 2001 From: Vadym Date: Mon, 20 May 2024 10:13:25 +0200 Subject: [PATCH 15/42] Block FFPE samples from Loqusdb upload (#3215) ### Added: - is_case_eligible_for_observations_upload to centralise upload eligibility checks. - Excluded FFPE samples from Loqusdb uploads (Balsamic & MIP-DNA). - Implemented exclusion of normal-only samples from Loqusdb uploads (Balsamic). ### Changed: - Simplified observations upload and delete workflows. - Removed mock classes from ObservationsAPI fixtures. - Reorganised and parameterised observations pytest. --- cg/apps/loqus.py | 2 +- cg/cli/delete/observations.py | 25 +- cg/cli/upload/observations/observations.py | 45 +- cg/cli/upload/observations/utils.py | 37 +- cg/constants/constants.py | 14 +- cg/constants/observations.py | 33 +- cg/constants/sample_sources.py | 19 +- .../observations/balsamic_observations_api.py | 99 ++-- .../observations/mip_dna_observations_api.py | 101 ++-- cg/meta/observations/observations_api.py | 100 ++-- cg/meta/workflow/analysis.py | 11 +- cg/meta/workflow/balsamic.py | 9 +- cg/store/filters/status_case_filters.py | 8 +- cg/store/models.py | 22 +- tests/apps/loqus/test_loqusdb_api.py | 60 +-- .../upload/test_cli_upload_observations.py | 92 +--- tests/conftest.py | 182 ++----- .../loqusdb_fixtures/loqusdb_api_fixtures.py | 57 ++ .../loqusdb_output_fixtures.py} | 88 +--- .../observations_fixtures/__init__.py | 0 .../observations_api_fixtures.py | 82 +++ .../observations_input_files_fixtures.py} | 48 +- tests/meta/observations/conftest.py | 94 ---- .../test_balsamic_observations_api.py | 190 +++++++ .../test_meta_upload_observations.py | 420 --------------- .../test_mip_dna_observations_api.py | 160 ++++++ .../observations/test_observations_api.py | 497 ++++++++++++++++++ tests/mocks/limsmock.py | 8 +- .../test_observations_input_files.py | 14 +- tests/store/conftest.py | 4 +- tests/utils/conftest.py | 2 +- 31 files changed, 1407 insertions(+), 1116 deletions(-) create mode 100644 tests/fixture_plugins/loqusdb_fixtures/loqusdb_api_fixtures.py rename tests/{apps/loqus/conftest.py => fixture_plugins/loqusdb_fixtures/loqusdb_output_fixtures.py} (64%) create mode 100644 tests/fixture_plugins/observations_fixtures/__init__.py create mode 100644 tests/fixture_plugins/observations_fixtures/observations_api_fixtures.py rename tests/{models/observations/conftest.py => fixture_plugins/observations_fixtures/observations_input_files_fixtures.py} (65%) delete mode 100644 tests/meta/observations/conftest.py create mode 100644 tests/meta/observations/test_balsamic_observations_api.py delete mode 100644 tests/meta/observations/test_meta_upload_observations.py create mode 100644 tests/meta/observations/test_mip_dna_observations_api.py create mode 100644 tests/meta/observations/test_observations_api.py diff --git a/cg/apps/loqus.py b/cg/apps/loqus.py index 7d948ac987..d302f72e9f 100644 --- a/cg/apps/loqus.py +++ b/cg/apps/loqus.py @@ -109,4 +109,4 @@ def get_nr_of_variants_in_file(self) -> dict[str, int]: return {"variants": nr_of_variants} def __repr__(self): - return f"LoqusdbAPI(binary_path={Path(self.binary_path).stem}, config_path={Path(self.config_path).stem})" + return f"LoqusdbAPI(binary_path={Path(self.binary_path).name}, config_path={Path(self.config_path).name})" diff --git a/cg/cli/delete/observations.py b/cg/cli/delete/observations.py index ac9e3bc4ec..4eb8831022 100644 --- a/cg/cli/delete/observations.py +++ b/cg/cli/delete/observations.py @@ -5,14 +5,13 @@ import click from sqlalchemy.orm import Query -from cg.cli.upload.observations.utils import get_observations_api, get_observations_case +from cg.cli.upload.observations.utils import get_observations_api from cg.cli.workflow.commands import ARGUMENT_CASE_ID, OPTION_LOQUSDB_SUPPORTED_WORKFLOW from cg.constants.constants import DRY_RUN, SKIP_CONFIRMATION, Workflow from cg.exc import CaseNotFoundError, LoqusdbError from cg.meta.observations.balsamic_observations_api import BalsamicObservationsAPI from cg.meta.observations.mip_dna_observations_api import MipDNAObservationsAPI from cg.models.cg_config import CGConfig -from cg.store.models import Case from cg.store.store import Store LOG = logging.getLogger(__name__) @@ -25,19 +24,15 @@ @click.pass_obj def delete_observations(context: CGConfig, case_id: str, dry_run: bool, yes: bool): """Delete a case from Loqusdb and reset the Loqusdb IDs in StatusDB.""" - - case: Case = get_observations_case(context, case_id, upload=False) observations_api: MipDNAObservationsAPI | BalsamicObservationsAPI = get_observations_api( - context, case + context=context, case_id=case_id, upload=False ) - if dry_run: - LOG.info(f"Dry run. This would delete all variants in Loqusdb for case: {case.internal_id}") + LOG.info(f"Dry run. This would delete all variants in Loqusdb for case: {case_id}") return - - LOG.info(f"This will delete all variants in Loqusdb for case: {case.internal_id}") + LOG.info(f"This will delete all variants in Loqusdb for case: {case_id}") if yes or click.confirm("Do you want to continue?", abort=True): - observations_api.delete_case(case) + observations_api.delete_case(case_id) @click.command("available-observations") @@ -49,22 +44,18 @@ def delete_available_observations( context: click.Context, workflow: Workflow | None, dry_run: bool, yes: bool ): """Delete available observation from Loqusdb.""" - status_db: Store = context.obj.status_db uploaded_observations: Query = status_db.observations_uploaded(workflow) - LOG.info( - f"This would delete observations for the following cases: {[case.internal_id for case in uploaded_observations]}" + f"This would delete observations for the following cases: " + f"{[case.internal_id for case in uploaded_observations]}" ) if yes or click.confirm("Do you want to continue?", abort=True): for case in uploaded_observations: try: LOG.info(f"Will delete observations for {case.internal_id}") context.invoke( - delete_observations, - case_id=case.internal_id, - dry_run=dry_run, - yes=yes, + delete_observations, case_id=case.internal_id, dry_run=dry_run, yes=yes ) except (CaseNotFoundError, LoqusdbError) as error: LOG.error(f"Error deleting observations for {case.internal_id}: {error}") diff --git a/cg/cli/upload/observations/observations.py b/cg/cli/upload/observations/observations.py index b8eea602b0..554e34f65a 100644 --- a/cg/cli/upload/observations/observations.py +++ b/cg/cli/upload/observations/observations.py @@ -1,27 +1,19 @@ """Code for uploading observations data via CLI.""" -import contextlib import logging from datetime import datetime import click -from pydantic.v1 import ValidationError from sqlalchemy.orm import Query -from cg.cli.upload.observations.utils import ( - get_observations_api, - get_observations_case_to_upload, -) -from cg.cli.workflow.commands import ( - ARGUMENT_CASE_ID, - OPTION_LOQUSDB_SUPPORTED_WORKFLOW, -) -from cg.constants.constants import Workflow, DRY_RUN -from cg.exc import CaseNotFoundError, LoqusdbError +from cg.cli.upload.observations.utils import get_observations_api +from cg.cli.workflow.commands import ARGUMENT_CASE_ID, OPTION_LOQUSDB_SUPPORTED_WORKFLOW +from cg.constants import EXIT_FAIL, EXIT_SUCCESS +from cg.constants.constants import DRY_RUN, Workflow +from cg.exc import CgError from cg.meta.observations.balsamic_observations_api import BalsamicObservationsAPI from cg.meta.observations.mip_dna_observations_api import MipDNAObservationsAPI from cg.models.cg_config import CGConfig -from cg.store.models import Case from cg.store.store import Store LOG = logging.getLogger(__name__) @@ -33,20 +25,17 @@ @click.pass_obj def upload_observations_to_loqusdb(context: CGConfig, case_id: str | None, dry_run: bool): """Upload observations from an analysis to Loqusdb.""" - click.echo(click.style("----------------- OBSERVATIONS -----------------")) - - with contextlib.suppress(LoqusdbError): - case: Case = get_observations_case_to_upload(context, case_id) + try: observations_api: MipDNAObservationsAPI | BalsamicObservationsAPI = get_observations_api( - context, case + context=context, case_id=case_id, upload=True ) - if dry_run: - LOG.info(f"Dry run. Would upload observations for {case.internal_id}.") + LOG.info(f"Dry run. Would upload observations for {case_id}.") return - - observations_api.upload(case) + observations_api.upload(case_id) + except CgError: + LOG.error(f"Could not upload {case_id} to Loqusdb") @click.command("available-observations") @@ -57,23 +46,23 @@ def upload_available_observations_to_loqusdb( context: click.Context, workflow: Workflow | None, dry_run: bool ): """Uploads the available observations to Loqusdb.""" - click.echo(click.style("----------------- AVAILABLE OBSERVATIONS -----------------")) - status_db: Store = context.obj.status_db - cases_to_upload: Query = status_db.observations_to_upload(workflow=workflow) + cases_to_upload: Query = status_db.observations_to_upload(workflow) if not cases_to_upload: LOG.error( f"There are no available cases to upload to Loqusdb for {workflow} ({datetime.now()})" ) return - + exit_code: int = EXIT_SUCCESS for case in cases_to_upload: try: LOG.info(f"Will upload observations for {case.internal_id}") context.invoke( upload_observations_to_loqusdb, case_id=case.internal_id, dry_run=dry_run ) - except (CaseNotFoundError, FileNotFoundError, ValidationError) as error: + except Exception as error: LOG.error(f"Error uploading observations for {case.internal_id}: {error}") - continue + exit_code = EXIT_FAIL + if exit_code: + raise click.Abort diff --git a/cg/cli/upload/observations/utils.py b/cg/cli/upload/observations/utils.py index ec340e19de..de1075865c 100644 --- a/cg/cli/upload/observations/utils.py +++ b/cg/cli/upload/observations/utils.py @@ -6,8 +6,7 @@ from cg.constants.constants import Workflow from cg.constants.observations import LOQUSDB_SUPPORTED_WORKFLOWS -from cg.constants.sequencing import SequencingMethod -from cg.exc import CaseNotFoundError, LoqusdbUploadCaseError +from cg.exc import CaseNotFoundError from cg.meta.observations.balsamic_observations_api import BalsamicObservationsAPI from cg.meta.observations.mip_dna_observations_api import MipDNAObservationsAPI from cg.models.cg_config import CGConfig @@ -17,7 +16,7 @@ LOG = logging.getLogger(__name__) -def get_observations_case(context: CGConfig, case_id: str, upload: bool) -> Case: +def get_observations_verified_case(context: CGConfig, case_id: str | None, upload: bool) -> Case: """Return a verified Loqusdb case.""" status_db: Store = context.status_db case: Case = status_db.get_case_by_internal_id(internal_id=case_id) @@ -32,41 +31,17 @@ def get_observations_case(context: CGConfig, case_id: str, upload: bool) -> Case LOG.info("Provide one of the following case IDs: ") for case in cases_to_process: LOG.info(f"{case.internal_id} ({case.data_analysis})") - raise CaseNotFoundError return case -def get_observations_case_to_upload(context: CGConfig, case_id: str) -> Case: - """Return a verified case ready to be uploaded to Loqusdb.""" - case: Case = get_observations_case(context, case_id, upload=True) - if not case.customer.loqus_upload: - LOG.error( - f"Customer {case.customer.internal_id} is not whitelisted for upload to Loqusdb. Canceling upload for " - f"case {case.internal_id}." - ) - raise LoqusdbUploadCaseError - return case - - def get_observations_api( - context: CGConfig, case: Case + context: CGConfig, case_id: str | None, upload: bool ) -> MipDNAObservationsAPI | BalsamicObservationsAPI: """Return an observations API given a specific case object.""" + case: Case = get_observations_verified_case(context=context, case_id=case_id, upload=upload) observations_apis = { - Workflow.MIP_DNA: MipDNAObservationsAPI(context, get_sequencing_method(case)), - Workflow.BALSAMIC: BalsamicObservationsAPI(context, get_sequencing_method(case)), + Workflow.MIP_DNA: MipDNAObservationsAPI(context), + Workflow.BALSAMIC: BalsamicObservationsAPI(context), } return observations_apis[case.data_analysis] - - -def get_sequencing_method(case: Case) -> SequencingMethod: - """Returns the sequencing method for the given case object.""" - analysis_types = [ - link.sample.application_version.application.analysis_type for link in case.links - ] - if len(set(analysis_types)) != 1: - LOG.error(f"Case {case.internal_id} has a mixed analysis type. Cancelling action.") - raise LoqusdbUploadCaseError - - return analysis_types[0] diff --git a/cg/constants/constants.py b/cg/constants/constants.py index f188596c95..da9801c946 100644 --- a/cg/constants/constants.py +++ b/cg/constants/constants.py @@ -1,6 +1,6 @@ """Constants for cg.""" -from enum import IntEnum, StrEnum +from enum import IntEnum, StrEnum, auto import click @@ -68,7 +68,11 @@ class CustomerId(StrEnum): CUST004: str = "cust004" CUST032: str = "cust032" CUST042: str = "cust042" + CUST110: str = "cust110" + CUST127: str = "cust127" CUST132: str = "cust132" + CUST143: str = "cust143" + CUST147: str = "cust147" CUST999: str = "cust999" @@ -92,6 +96,14 @@ class AnalysisType(StrEnum): OTHER: str = "other" +class CancerAnalysisType(StrEnum): + TUMOR_NORMAL = auto() + TUMOR_NORMAL_PANEL = auto() + TUMOR_NORMAL_WGS = auto() + TUMOR_PANEL = auto() + TUMOR_WGS = auto() + + class PrepCategory(StrEnum): COVID: str = "cov" MICROBIAL: str = "mic" diff --git a/cg/constants/observations.py b/cg/constants/observations.py index d68b28d0a2..f00b9a45f3 100644 --- a/cg/constants/observations.py +++ b/cg/constants/observations.py @@ -2,30 +2,23 @@ from enum import Enum, StrEnum -from cg.constants.constants import Workflow +from cg.constants.constants import CancerAnalysisType, CustomerId, Workflow from cg.constants.sequencing import SequencingMethod LOQUSDB_ID = "_id" LOQUSDB_SUPPORTED_WORKFLOWS = [Workflow.MIP_DNA, Workflow.BALSAMIC] -LOQUSDB_MIP_SEQUENCING_METHODS = [SequencingMethod.WGS, SequencingMethod.WES] -LOQUSDB_BALSAMIC_SEQUENCING_METHODS = [SequencingMethod.WGS] - - -class LoqusdbMipCustomers(StrEnum): - """Loqusdb Rare Disease customers.""" - - KLINISK_GENETIK: str = "cust002" - CMMS: str = "cust003" - KLINISK_IMMUNOLOGI: str = "cust004" - - -class LoqusdbBalsamicCustomers(StrEnum): - """Loqusdb Balsamic customers.""" - - AML: str = "cust110" - BTB_GMS: str = "cust127" - ONKPAT_HAGLUND: str = "cust143" - BTB_GMS_LINKOPING: str = "cust147" +LOQUSDB_RARE_DISEASE_CUSTOMERS = [CustomerId.CUST002, CustomerId.CUST003, CustomerId.CUST004] +LOQUSDB_CANCER_CUSTOMERS = [ + CustomerId.CUST110, + CustomerId.CUST127, + CustomerId.CUST143, + CustomerId.CUST147, +] +LOQUSDB_RARE_DISEASE_SEQUENCING_METHODS = [SequencingMethod.WGS, SequencingMethod.WES] +LOQUSDB_CANCER_SEQUENCING_METHODS = [ + CancerAnalysisType.TUMOR_WGS, + CancerAnalysisType.TUMOR_NORMAL_WGS, +] class LoqusdbInstance(StrEnum): diff --git a/cg/constants/sample_sources.py b/cg/constants/sample_sources.py index 383c3ed8e1..cf5ffa6b6c 100644 --- a/cg/constants/sample_sources.py +++ b/cg/constants/sample_sources.py @@ -1,4 +1,4 @@ -"""Constants that specify sample sources""" +"""Constants that specify sample sources.""" from enum import StrEnum @@ -13,6 +13,7 @@ "unknown", "other", ) + ANALYSIS_SOURCES = ( "blood", "buccal swab", @@ -33,4 +34,20 @@ class SourceType(StrEnum): + BLOOD: str = "blood" + BONE_MARROW: str = "bone marrow" + BUCCAL_SWAB: str = "buccal swab" + CELL_FREE_DNA: str = "cell-free DNA" + CELL_LINE: str = "cell line" + CYTOLOGY: str = "cytology (not fixed/fresh)" + CYTOLOGY_FFPE: str = "cytology (FFPE)" + FFPE: str = "FFPE" + FIBROBLAST: str = "fibroblast" + MUSCLE: str = "muscle" + NAIL: str = "nail" + OTHER: str = "other" + SALIVA: str = "saliva" + SKIN: str = "skin" + TISSUE: str = "tissue (fresh frozen)" + TISSUE_FFPE: str = "tissue (FFPE)" UNKNOWN: str = "unknown" diff --git a/cg/meta/observations/balsamic_observations_api.py b/cg/meta/observations/balsamic_observations_api.py index f76aa93b78..8d176196e4 100644 --- a/cg/meta/observations/balsamic_observations_api.py +++ b/cg/meta/observations/balsamic_observations_api.py @@ -6,21 +6,18 @@ from housekeeper.store.models import File, Version from cg.apps.loqus import LoqusdbAPI +from cg.constants.constants import CancerAnalysisType, CustomerId from cg.constants.observations import ( - LOQUSDB_BALSAMIC_SEQUENCING_METHODS, + LOQUSDB_CANCER_CUSTOMERS, + LOQUSDB_CANCER_SEQUENCING_METHODS, LOQUSDB_ID, BalsamicLoadParameters, BalsamicObservationsAnalysisTag, - LoqusdbBalsamicCustomers, LoqusdbInstance, ) -from cg.constants.sequencing import SequencingMethod -from cg.exc import ( - CaseNotFoundError, - LoqusdbDuplicateRecordError, - LoqusdbUploadCaseError, -) +from cg.exc import CaseNotFoundError, LoqusdbDuplicateRecordError from cg.meta.observations.observations_api import ObservationsAPI +from cg.meta.workflow.balsamic import BalsamicAnalysisAPI from cg.models.cg_config import CGConfig from cg.models.observations.input_files import BalsamicObservationsInputFiles from cg.store.models import Case @@ -32,31 +29,54 @@ class BalsamicObservationsAPI(ObservationsAPI): """API to manage Balsamic observations.""" - def __init__(self, config: CGConfig, sequencing_method: SequencingMethod): - super().__init__(config) - self.sequencing_method: SequencingMethod = sequencing_method + def __init__(self, config: CGConfig): + self.analysis_api = BalsamicAnalysisAPI(config) + super().__init__(config=config, analysis_api=self.analysis_api) self.loqusdb_somatic_api: LoqusdbAPI = self.get_loqusdb_api(LoqusdbInstance.SOMATIC) self.loqusdb_tumor_api: LoqusdbAPI = self.get_loqusdb_api(LoqusdbInstance.TUMOR) - def load_observations(self, case: Case, input_files: BalsamicObservationsInputFiles) -> None: - """Load observation counts to Loqusdb for a Balsamic case.""" - if self.sequencing_method not in LOQUSDB_BALSAMIC_SEQUENCING_METHODS: - LOG.error( - f"Sequencing method {self.sequencing_method} is not supported by Loqusdb. Cancelling upload." - ) - raise LoqusdbUploadCaseError + @property + def loqusdb_customers(self) -> list[CustomerId]: + """Customers that are eligible for cancer Loqusdb uploads.""" + return LOQUSDB_CANCER_CUSTOMERS + + @property + def loqusdb_sequencing_methods(self) -> list[str]: + """Return sequencing methods that are eligible for cancer Loqusdb uploads.""" + return LOQUSDB_CANCER_SEQUENCING_METHODS + + def is_analysis_type_eligible_for_observations_upload(self, case_id) -> bool: + """Return whether the cancer analysis type is eligible for cancer Loqusdb uploads.""" + if self.analysis_api.is_analysis_normal_only(case_id): + LOG.error(f"Normal only analysis {case_id} is not supported for Loqusdb uploads") + return False + return True + + def is_case_eligible_for_observations_upload(self, case: Case) -> bool: + """Return whether a cancer case is eligible for observations upload.""" + return all( + [ + self.is_customer_eligible_for_observations_upload(case.customer.internal_id), + self.is_sequencing_method_eligible_for_observations_upload(case.internal_id), + self.is_analysis_type_eligible_for_observations_upload(case.internal_id), + self.is_sample_source_eligible_for_observations_upload(case.internal_id), + ] + ) + + def load_observations(self, case: Case) -> None: + """ + Load observation counts to Loqusdb for a Balsamic case. + Raises: + LoqusdbDuplicateRecordError: If case has already been uploaded. + """ loqusdb_upload_apis: list[LoqusdbAPI] = [self.loqusdb_somatic_api, self.loqusdb_tumor_api] for loqusdb_api in loqusdb_upload_apis: - if self.is_duplicate( - case=case, - loqusdb_api=loqusdb_api, - profile_vcf_path=None, - profile_threshold=None, - ): + if self.is_duplicate(case=case, loqusdb_api=loqusdb_api): LOG.error(f"Case {case.internal_id} has already been uploaded to Loqusdb") raise LoqusdbDuplicateRecordError + input_files: BalsamicObservationsInputFiles = self.get_observations_input_files(case) for loqusdb_api in loqusdb_upload_apis: self.load_cancer_observations( case=case, input_files=input_files, loqusdb_api=loqusdb_api @@ -70,8 +90,11 @@ def load_cancer_observations( self, case: Case, input_files: BalsamicObservationsInputFiles, loqusdb_api: LoqusdbAPI ) -> None: """Load cancer observations to a specific Loqusdb API.""" - is_somatic_db: bool = "somatic" in str(loqusdb_api.config_path) - is_paired_analysis: bool = len(self.store.get_samples_by_case_id(case.internal_id)) == 2 + is_somatic_db: bool = LoqusdbInstance.SOMATIC in str(loqusdb_api.config_path) + is_paired_analysis: bool = ( + CancerAnalysisType.TUMOR_NORMAL + in self.analysis_api.get_data_analysis_type(case.internal_id) + ) if is_somatic_db: if not is_paired_analysis: return @@ -96,10 +119,10 @@ def load_cancer_observations( ) LOG.info(f"Uploaded {load_output['variants']} variants to {repr(loqusdb_api)}") - def extract_observations_files_from_hk( - self, hk_version: Version + def get_observations_files_from_hk( + self, hk_version: Version, case_id: str = None ) -> BalsamicObservationsInputFiles: - """Extract observations files given a housekeeper version for cancer.""" + """Return observations files given a Housekeeper version for cancer.""" input_files: dict[str, File] = { "snv_germline_vcf_path": self.housekeeper_api.files( version=hk_version.id, tags=[BalsamicObservationsAnalysisTag.SNV_GERMLINE_VCF] @@ -116,21 +139,15 @@ def extract_observations_files_from_hk( } return BalsamicObservationsInputFiles(**get_full_path_dictionary(input_files)) - def delete_case(self, case: Case) -> None: + def delete_case(self, case_id: str) -> None: """Delete cancer case observations from Loqusdb.""" + case: Case = self.store.get_case_by_internal_id(internal_id=case_id) loqusdb_apis: list[LoqusdbAPI] = [self.loqusdb_somatic_api, self.loqusdb_tumor_api] for loqusdb_api in loqusdb_apis: - if not loqusdb_api.get_case(case.internal_id): - LOG.error( - f"Case {case.internal_id} could not be found in Loqusdb. Skipping case deletion." - ) + if not loqusdb_api.get_case(case_id): + LOG.error(f"Case {case_id} could not be found in Loqusdb. Skipping case deletion.") raise CaseNotFoundError - for loqusdb_api in loqusdb_apis: - loqusdb_api.delete_case(case.internal_id) + loqusdb_api.delete_case(case_id) self.update_statusdb_loqusdb_id(samples=case.samples, loqusdb_id=None) - LOG.info(f"Removed observations for case {case.internal_id} from Loqusdb") - - def get_loqusdb_customers(self) -> LoqusdbBalsamicCustomers: - """Returns the customers that are entitled to Cancer Loqusdb uploads.""" - return LoqusdbBalsamicCustomers + LOG.info(f"Removed observations for case {case_id} from Loqusdb") diff --git a/cg/meta/observations/mip_dna_observations_api.py b/cg/meta/observations/mip_dna_observations_api.py index be5407fcc2..49a01c7063 100644 --- a/cg/meta/observations/mip_dna_observations_api.py +++ b/cg/meta/observations/mip_dna_observations_api.py @@ -4,22 +4,19 @@ from housekeeper.store.models import File, Version -from cg.apps.loqus import LoqusdbAPI +from cg.constants.constants import CustomerId, SampleType from cg.constants.observations import ( LOQUSDB_ID, - LOQUSDB_MIP_SEQUENCING_METHODS, + LOQUSDB_RARE_DISEASE_CUSTOMERS, + LOQUSDB_RARE_DISEASE_SEQUENCING_METHODS, LoqusdbInstance, - LoqusdbMipCustomers, MipDNALoadParameters, MipDNAObservationsAnalysisTag, ) from cg.constants.sequencing import SequencingMethod -from cg.exc import ( - CaseNotFoundError, - LoqusdbDuplicateRecordError, - LoqusdbUploadCaseError, -) +from cg.exc import CaseNotFoundError, LoqusdbDuplicateRecordError from cg.meta.observations.observations_api import ObservationsAPI +from cg.meta.workflow.mip_dna import MipDNAAnalysisAPI from cg.models.cg_config import CGConfig from cg.models.observations.input_files import MipDNAObservationsInputFiles from cg.store.models import Case @@ -31,31 +28,58 @@ class MipDNAObservationsAPI(ObservationsAPI): """API to manage MIP-DNA observations.""" - def __init__(self, config: CGConfig, sequencing_method: SequencingMethod): - super().__init__(config) - self.sequencing_method: SequencingMethod = sequencing_method - self.loqusdb_api: LoqusdbAPI = self.get_loqusdb_api(self.get_loqusdb_instance()) + def __init__(self, config: CGConfig): + self.analysis_api = MipDNAAnalysisAPI(config) + super().__init__(config=config, analysis_api=self.analysis_api) + self.loqusdb_api = None - def get_loqusdb_instance(self) -> LoqusdbInstance: - """Return the Loqusdb instance associated to the sequencing method.""" - if self.sequencing_method not in LOQUSDB_MIP_SEQUENCING_METHODS: - LOG.error( - f"Sequencing method {self.sequencing_method} is not supported by Loqusdb. Cancelling upload." - ) - raise LoqusdbUploadCaseError + @property + def loqusdb_customers(self) -> list[CustomerId]: + """Customers that are eligible for rare disease Loqusdb uploads.""" + return LOQUSDB_RARE_DISEASE_CUSTOMERS + + @property + def loqusdb_sequencing_methods(self) -> list[str]: + """Sequencing methods that are eligible for cancer Loqusdb uploads.""" + return LOQUSDB_RARE_DISEASE_SEQUENCING_METHODS + + @staticmethod + def is_sample_type_eligible_for_observations_upload(case: Case) -> bool: + """Return whether a rare disease case is free of tumor samples.""" + if case.tumour_samples: + LOG.error(f"Sample type {SampleType.TUMOR} is not supported for Loqusdb uploads") + return False + return True + + def is_case_eligible_for_observations_upload(self, case: Case) -> bool: + """Return whether a rare disease case is eligible for observations upload.""" + return all( + [ + self.is_customer_eligible_for_observations_upload(case.customer.internal_id), + self.is_sequencing_method_eligible_for_observations_upload(case.internal_id), + self.is_sample_type_eligible_for_observations_upload(case), + self.is_sample_source_eligible_for_observations_upload(case.internal_id), + ] + ) + def set_loqusdb_instance(self, case_id: str) -> None: + """Return the Loqusdb instance associated to the sequencing method.""" + sequencing_method: SequencingMethod = self.analysis_api.get_data_analysis_type(case_id) loqusdb_instances: dict[SequencingMethod, LoqusdbInstance] = { SequencingMethod.WGS: LoqusdbInstance.WGS, SequencingMethod.WES: LoqusdbInstance.WES, } - return loqusdb_instances[self.sequencing_method] + self.loqusdb_api = self.get_loqusdb_api(loqusdb_instances[sequencing_method]) - def load_observations(self, case: Case, input_files: MipDNAObservationsInputFiles) -> None: - """Load observation counts to Loqusdb for a MIP-DNA case.""" - if case.tumour_samples: - LOG.error(f"Case {case.internal_id} has tumour samples. Cancelling upload.") - raise LoqusdbUploadCaseError + def load_observations(self, case: Case) -> None: + """ + Load observation counts to Loqusdb for a MIP-DNA case. + Raises: + LoqusdbDuplicateRecordError: If case has already been uploaded. + """ + self.set_loqusdb_instance(case.internal_id) + input_files: MipDNAObservationsInputFiles = self.get_observations_input_files(case) if self.is_duplicate( case=case, loqusdb_api=self.loqusdb_api, @@ -81,10 +105,10 @@ def load_observations(self, case: Case, input_files: MipDNAObservationsInputFile self.update_statusdb_loqusdb_id(samples=case.samples, loqusdb_id=loqusdb_id) LOG.info(f"Uploaded {load_output['variants']} variants to {repr(self.loqusdb_api)}") - def extract_observations_files_from_hk( - self, hk_version: Version + def get_observations_files_from_hk( + self, hk_version: Version, case_id: str ) -> MipDNAObservationsInputFiles: - """Extract observations files given a housekeeper version for rare diseases.""" + """Return observations files given a Housekeeper version for rare diseases.""" input_files: dict[str, File] = { "snv_vcf_path": self.housekeeper_api.files( version=hk_version.id, tags=[MipDNAObservationsAnalysisTag.SNV_VCF] @@ -93,7 +117,7 @@ def extract_observations_files_from_hk( self.housekeeper_api.files( version=hk_version.id, tags=[MipDNAObservationsAnalysisTag.SV_VCF] ).first() - if self.sequencing_method == SequencingMethod.WGS + if self.analysis_api.get_data_analysis_type(case_id) == SequencingMethod.WGS else None ), "profile_vcf_path": self.housekeeper_api.files( @@ -105,18 +129,13 @@ def extract_observations_files_from_hk( } return MipDNAObservationsInputFiles(**get_full_path_dictionary(input_files)) - def delete_case(self, case: Case) -> None: + def delete_case(self, case_id: str) -> None: """Delete rare disease case observations from Loqusdb.""" - if not self.loqusdb_api.get_case(case.internal_id): - LOG.error( - f"Case {case.internal_id} could not be found in Loqusdb. Skipping case deletion." - ) + case: Case = self.store.get_case_by_internal_id(internal_id=case_id) + self.set_loqusdb_instance(case_id) + if not self.loqusdb_api.get_case(case_id): + LOG.error(f"Case {case_id} could not be found in Loqusdb. Skipping case deletion.") raise CaseNotFoundError - - self.loqusdb_api.delete_case(case.internal_id) + self.loqusdb_api.delete_case(case_id) self.update_statusdb_loqusdb_id(samples=case.samples, loqusdb_id=None) - LOG.info(f"Removed observations for case {case.internal_id} from {repr(self.loqusdb_api)}") - - def get_loqusdb_customers(self) -> LoqusdbMipCustomers: - """Returns the customers that are entitled to Rare Disease Loqusdb uploads.""" - return LoqusdbMipCustomers + LOG.info(f"Removed observations for case {case_id} from {repr(self.loqusdb_api)}") diff --git a/cg/meta/observations/observations_api.py b/cg/meta/observations/observations_api.py index d5bd9aaaaf..9a7cd6a8f3 100644 --- a/cg/meta/observations/observations_api.py +++ b/cg/meta/observations/observations_api.py @@ -8,18 +8,18 @@ from cg.apps.housekeeper.hk import HousekeeperAPI from cg.apps.loqus import LoqusdbAPI -from cg.constants.observations import ( - LoqusdbBalsamicCustomers, - LoqusdbInstance, - LoqusdbMipCustomers, -) +from cg.constants.constants import CustomerId +from cg.constants.observations import LoqusdbInstance +from cg.constants.sample_sources import SourceType +from cg.constants.sequencing import SequencingMethod from cg.exc import LoqusdbUploadCaseError +from cg.meta.workflow.analysis import AnalysisAPI from cg.models.cg_config import CGConfig, CommonAppConfig from cg.models.observations.input_files import ( BalsamicObservationsInputFiles, MipDNAObservationsInputFiles, ) -from cg.store.models import Analysis, Case, Customer +from cg.store.models import Analysis, Case from cg.store.store import Store LOG = logging.getLogger(__name__) @@ -28,21 +28,31 @@ class ObservationsAPI: """API to manage Loqusdb observations.""" - def __init__(self, config: CGConfig): + def __init__(self, config: CGConfig, analysis_api: AnalysisAPI): self.store: Store = config.status_db self.housekeeper_api: HousekeeperAPI = config.housekeeper_api + self.analysis_api: AnalysisAPI = analysis_api self.loqusdb_config: CommonAppConfig = config.loqusdb self.loqusdb_wes_config: CommonAppConfig = config.loqusdb_wes self.loqusdb_somatic_config: CommonAppConfig = config.loqusdb_somatic self.loqusdb_tumor_config: CommonAppConfig = config.loqusdb_tumor - def upload(self, case: Case) -> None: - """Upload observations to Loqusdb.""" - self.check_customer_loqusdb_permissions(case.customer) - input_files: MipDNAObservationsInputFiles | BalsamicObservationsInputFiles = ( - self.get_observations_input_files(case) + def upload(self, case_id: str) -> None: + """ + Upload observations to Loqusdb. + + Raises: + LoqusdbUploadCaseError: If case is not eligible for Loqusdb uploads + """ + case: Case = self.store.get_case_by_internal_id(internal_id=case_id) + is_case_eligible_for_observations_upload: bool = ( + self.is_case_eligible_for_observations_upload(case) ) - self.load_observations(case=case, input_files=input_files) + if is_case_eligible_for_observations_upload: + self.load_observations(case=case) + else: + LOG.error(f"Case {case.internal_id} is not eligible for observations upload") + raise LoqusdbUploadCaseError def get_observations_input_files( self, case: Case @@ -51,7 +61,7 @@ def get_observations_input_files( analysis: Analysis = case.analyses[0] analysis_date: datetime = analysis.started_at or analysis.completed_at hk_version: Version = self.housekeeper_api.version(analysis.case.internal_id, analysis_date) - return self.extract_observations_files_from_hk(hk_version) + return self.get_observations_files_from_hk(hk_version=hk_version, case_id=case.internal_id) def get_loqusdb_api(self, loqusdb_instance: LoqusdbInstance) -> LoqusdbAPI: """Returns a Loqusdb API for the given Loqusdb instance.""" @@ -79,8 +89,8 @@ def get_loqusdb_api(self, loqusdb_instance: LoqusdbInstance) -> LoqusdbAPI: def is_duplicate( case: Case, loqusdb_api: LoqusdbAPI, - profile_vcf_path: Path | None, - profile_threshold: float | None, + profile_vcf_path: Path | None = None, + profile_threshold: float | None = None, ) -> bool: """Check if a case has already been uploaded to Loqusdb.""" loqusdb_case: dict = loqusdb_api.get_case(case_id=case.internal_id) @@ -99,31 +109,53 @@ def update_statusdb_loqusdb_id(self, samples: list[Case], loqusdb_id: str | None sample.loqusdb_id = loqusdb_id self.store.session.commit() - def check_customer_loqusdb_permissions(self, customer: Customer) -> None: - """Verifies that the customer is whitelisted for Loqusdb uploads.""" - if customer.internal_id not in [cust_id for cust_id in self.get_loqusdb_customers()]: - LOG.error( - f"Customer {customer.internal_id} is not whitelisted for Loqusdb uploads. Cancelling upload." - ) - raise LoqusdbUploadCaseError - LOG.info(f"Valid customer {customer.internal_id} for Loqusdb uploads") + def is_customer_eligible_for_observations_upload(self, customer_id: str) -> bool: + """Return whether the customer has been whitelisted for uploading observations.""" + if customer_id not in self.loqusdb_customers: + LOG.error(f"Customer {customer_id} is not whitelisted for Loqusdb uploads") + return False + return True + + def is_sequencing_method_eligible_for_observations_upload(self, case_id: str) -> bool: + """Return whether a sequencing method is valid for observations upload.""" + sequencing_method: SequencingMethod | None = self.analysis_api.get_data_analysis_type( + case_id + ) + if sequencing_method not in self.loqusdb_sequencing_methods: + LOG.error(f"Sequencing method {sequencing_method} is not supported by Loqusdb uploads") + return False + return True + + def is_sample_source_eligible_for_observations_upload(self, case_id: str) -> bool: + """Check if the sample source is FFPE.""" + source_type: str | None = self.analysis_api.get_case_source_type(case_id) + if source_type and SourceType.FFPE.lower() not in source_type.lower(): + return True + LOG.error(f"Source type {source_type} is not supported for Loqusdb uploads") + return False + + @property + def loqusdb_customers(self) -> list[CustomerId]: + """Customers that are eligible for Loqusdb uploads.""" + raise NotImplementedError - def get_loqusdb_customers(self) -> LoqusdbMipCustomers | LoqusdbBalsamicCustomers: - """Returns the customers that are entitled to Loqusdb uploads.""" + @property + def loqusdb_sequencing_methods(self) -> list[str]: + """Sequencing methods that are eligible for Loqusdb uploads.""" raise NotImplementedError - def load_observations( - self, - case: Case, - input_files: MipDNAObservationsInputFiles | BalsamicObservationsInputFiles, - ) -> None: + def load_observations(self, case: Case) -> None: """Load observation counts to Loqusdb.""" raise NotImplementedError - def extract_observations_files_from_hk( - self, hk_version: Version + def is_case_eligible_for_observations_upload(self, case: Case) -> bool: + """Return whether a case is eligible for observations upload.""" + raise NotImplementedError + + def get_observations_files_from_hk( + self, hk_version: Version, case_id: str ) -> MipDNAObservationsInputFiles | BalsamicObservationsInputFiles: - """Extract observations files given a housekeeper version.""" + """Return observations files given a Housekeeper version.""" raise NotImplementedError def delete_case(self, case: Case) -> None: diff --git a/cg/meta/workflow/analysis.py b/cg/meta/workflow/analysis.py index 72bcbc1cce..46e5de2455 100644 --- a/cg/meta/workflow/analysis.py +++ b/cg/meta/workflow/analysis.py @@ -190,17 +190,18 @@ def get_case_application_type(self, case_id: str) -> str: return application_types.pop() def get_case_source_type(self, case_id: str) -> str | None: - """Returns the source type for samples in a case. + """ + Return the sample source type of a case. + Raises: - CgError: If different sources are set for the samples linked to a case.""" + CgError: If different sources are set for the samples linked to a case. + """ sample_ids: Iterator[str] = self.status_db.get_sample_ids_by_case_id(case_id=case_id) source_types: set[str | None] = { - self.lims_api.get_source(lims_id=sample_id) for sample_id in sample_ids + self.lims_api.get_source(sample_id) for sample_id in sample_ids } - if len(source_types) > 1: raise CgError(f"Different source types found for case: {case_id} ({source_types})") - return source_types.pop() def has_case_only_exome_samples(self, case_id: str) -> bool: diff --git a/cg/meta/workflow/balsamic.py b/cg/meta/workflow/balsamic.py index 841bfecafe..e2c2ae0fea 100644 --- a/cg/meta/workflow/balsamic.py +++ b/cg/meta/workflow/balsamic.py @@ -141,7 +141,7 @@ def get_bundle_deliverables_type(self, case_id: str) -> str: if application_type != "wgs": application_type = "panel" analysis_type = "_".join([sample_type, application_type]) - LOG.info(f"Found analysis type {analysis_type}") + LOG.debug(f"Found analysis type {analysis_type}") return analysis_type def get_sample_fastq_destination_dir(self, case: Case, sample: Sample = None) -> Path: @@ -651,3 +651,10 @@ def get_variant_callers(self, case_id: str) -> list[str]: def get_data_analysis_type(self, case_id: str) -> str | None: """Return data analysis type carried out.""" return self.get_bundle_deliverables_type(case_id) + + def is_analysis_normal_only(self, case_id: str) -> bool: + """Return whether the analysis is normal only.""" + case: Case = self.status_db.get_case_by_internal_id(internal_id=case_id) + if case.non_tumour_samples and not case.tumour_samples: + return True + return False diff --git a/cg/store/filters/status_case_filters.py b/cg/store/filters/status_case_filters.py index 2bfb84af50..ef316002d9 100644 --- a/cg/store/filters/status_case_filters.py +++ b/cg/store/filters/status_case_filters.py @@ -8,8 +8,8 @@ from cg.constants import REPORT_SUPPORTED_DATA_DELIVERY from cg.constants.constants import CaseActions, DataDelivery, Workflow from cg.constants.observations import ( - LOQUSDB_BALSAMIC_SEQUENCING_METHODS, - LOQUSDB_MIP_SEQUENCING_METHODS, + LOQUSDB_CANCER_SEQUENCING_METHODS, + LOQUSDB_RARE_DISEASE_SEQUENCING_METHODS, LOQUSDB_SUPPORTED_WORKFLOWS, ) from cg.store.models import Analysis, Application, Case, Customer, Sample @@ -148,8 +148,8 @@ def filter_cases_with_loqusdb_supported_sequencing_method( ) -> Query: """Filter cases with Loqusdb supported sequencing method.""" supported_sequencing_methods = { - Workflow.MIP_DNA: LOQUSDB_MIP_SEQUENCING_METHODS, - Workflow.BALSAMIC: LOQUSDB_BALSAMIC_SEQUENCING_METHODS, + Workflow.MIP_DNA: LOQUSDB_RARE_DISEASE_SEQUENCING_METHODS, + Workflow.BALSAMIC: LOQUSDB_CANCER_SEQUENCING_METHODS, } return ( cases.filter(Application.prep_category.in_(supported_sequencing_methods[workflow])) diff --git a/cg/store/models.py b/cg/store/models.py index 117f12115c..9f05b7b314 100644 --- a/cg/store/models.py +++ b/cg/store/models.py @@ -510,36 +510,26 @@ def __str__(self) -> str: @property def samples(self) -> list["Sample"]: """Return case samples.""" - return self._get_samples + return [link.sample for link in self.links] @property def sample_ids(self) -> list[str]: """Return a list of internal ids of the case samples.""" - return [sample.internal_id for sample in self._get_samples] - - @property - def _get_samples(self) -> list["Sample"]: - """Extract samples from a case.""" - return [link.sample for link in self.links] + return [sample.internal_id for sample in self.samples] @property def tumour_samples(self) -> list["Sample"]: """Return tumour samples.""" - return self._get_tumour_samples + return [link.sample for link in self.links if link.sample.is_tumour] @property - def _get_tumour_samples(self) -> list["Sample"]: - """Extract tumour samples.""" - return [link.sample for link in self.links if link.sample.is_tumour] + def non_tumour_samples(self) -> list["Sample"]: + """Return non-tumour samples.""" + return [link.sample for link in self.links if not link.sample.is_tumour] @property def loqusdb_uploaded_samples(self) -> list["Sample"]: """Return uploaded samples to Loqusdb.""" - return self._get_loqusdb_uploaded_samples - - @property - def _get_loqusdb_uploaded_samples(self) -> list["Sample"]: - """Extract samples uploaded to Loqusdb.""" return [link.sample for link in self.links if link.sample.loqusdb_id] @property diff --git a/tests/apps/loqus/test_loqusdb_api.py b/tests/apps/loqus/test_loqusdb_api.py index 08a58625fb..dbc7ce7562 100644 --- a/tests/apps/loqus/test_loqusdb_api.py +++ b/tests/apps/loqus/test_loqusdb_api.py @@ -15,12 +15,12 @@ from cg.models.observations.input_files import MipDNAObservationsInputFiles -def test_instantiate(cg_config_locusdb: CGConfig): +def test_instantiate(cg_context: CGConfig): """Test instantiation of Loqusdb API.""" # GIVEN a Loqusdb binary and config paths - binary_path: str = cg_config_locusdb.loqusdb.binary_path - config_path: str = cg_config_locusdb.loqusdb.config_path + binary_path: str = cg_context.loqusdb.binary_path + config_path: str = cg_context.loqusdb.config_path # WHEN instantiating a Loqusdb api loqusdb_api = LoqusdbAPI(binary_path=binary_path, config_path=config_path) @@ -34,9 +34,9 @@ def test_instantiate(cg_config_locusdb: CGConfig): def test_load( case_id: str, loqusdb_api: LoqusdbAPI, - observations_input_files: MipDNAObservationsInputFiles, + mip_dna_observations_input_files: MipDNAObservationsInputFiles, loqusdb_load_stderr: bytes, - nr_of_loaded_variants: int, + number_of_loaded_variants: int, ): """Test loading of case to Loqusdb.""" @@ -46,23 +46,23 @@ def test_load( loqusdb_api.process.stderr = loqusdb_load_stderr.decode("utf-8") output: dict = loqusdb_api.load( case_id=case_id, - snv_vcf_path=observations_input_files.snv_vcf_path, - sv_vcf_path=observations_input_files.sv_vcf_path, - profile_vcf_path=observations_input_files.profile_vcf_path, - family_ped_path=observations_input_files.family_ped_path, + snv_vcf_path=mip_dna_observations_input_files.snv_vcf_path, + sv_vcf_path=mip_dna_observations_input_files.sv_vcf_path, + profile_vcf_path=mip_dna_observations_input_files.profile_vcf_path, + family_ped_path=mip_dna_observations_input_files.family_ped_path, gq_threshold=MipDNALoadParameters.GQ_THRESHOLD.value, hard_threshold=MipDNALoadParameters.HARD_THRESHOLD.value, soft_threshold=MipDNALoadParameters.SOFT_THRESHOLD.value, ) # THEN assert that the number of variants is the expected one - assert output["variants"] == nr_of_loaded_variants + assert output["variants"] == number_of_loaded_variants def test_load_parameters( case_id: str, loqusdb_api: LoqusdbAPI, - observations_input_files: MipDNAObservationsInputFiles, + mip_dna_observations_input_files: MipDNAObservationsInputFiles, loqusdb_load_stderr: bytes, caplog: LogCaptureFixture, ): @@ -75,10 +75,10 @@ def test_load_parameters( loqusdb_api.process.stderr = loqusdb_load_stderr.decode("utf-8") loqusdb_api.load( case_id=case_id, - snv_vcf_path=observations_input_files.snv_vcf_path, - sv_vcf_path=observations_input_files.sv_vcf_path, - profile_vcf_path=observations_input_files.profile_vcf_path, - family_ped_path=observations_input_files.family_ped_path, + snv_vcf_path=mip_dna_observations_input_files.snv_vcf_path, + sv_vcf_path=mip_dna_observations_input_files.sv_vcf_path, + profile_vcf_path=mip_dna_observations_input_files.profile_vcf_path, + family_ped_path=mip_dna_observations_input_files.family_ped_path, gq_threshold=MipDNALoadParameters.GQ_THRESHOLD.value, hard_threshold=MipDNALoadParameters.HARD_THRESHOLD.value, soft_threshold=None, @@ -86,20 +86,20 @@ def test_load_parameters( # THEN assert that the expected params are included in the call assert f"--case-id {case_id}" in caplog.text - assert f"--variant-file {observations_input_files.snv_vcf_path}" in caplog.text - assert f"--sv-variants" not in caplog.text - assert f"--check-profile {observations_input_files.profile_vcf_path}" in caplog.text - assert f"--family-file {observations_input_files.family_ped_path}" in caplog.text - assert f"--max-window" not in caplog.text + assert f"--variant-file {mip_dna_observations_input_files.snv_vcf_path}" in caplog.text + assert "--sv-variants" not in caplog.text + assert f"--check-profile {mip_dna_observations_input_files.profile_vcf_path}" in caplog.text + assert f"--family-file {mip_dna_observations_input_files.family_ped_path}" in caplog.text + assert "--max-window" not in caplog.text assert f"--gq-threshold {MipDNALoadParameters.GQ_THRESHOLD.value}" in caplog.text assert f"--hard-threshold {MipDNALoadParameters.HARD_THRESHOLD.value}" in caplog.text - assert f"--soft-threshold" not in caplog.text + assert "--soft-threshold" not in caplog.text def test_load_exception( case_id: str, loqusdb_api_exception: LoqusdbAPI, - observations_input_files: MipDNAObservationsInputFiles, + mip_dna_observations_input_files: MipDNAObservationsInputFiles, ): """Test Loqusdb load command with a failed output.""" @@ -109,7 +109,7 @@ def test_load_exception( # THEN an error should be raised with pytest.raises(CalledProcessError): - loqusdb_api_exception.load(case_id, observations_input_files.snv_vcf_path) + loqusdb_api_exception.load(case_id, mip_dna_observations_input_files.snv_vcf_path) def test_get_case(case_id: str, loqusdb_api: LoqusdbAPI, loqusdb_case_output: bytes): @@ -143,7 +143,7 @@ def test_get_case_non_existing(case_id: str, loqusdb_api: LoqusdbAPI, caplog: Lo def test_get_duplicate( loqusdb_api: LoqusdbAPI, loqusdb_duplicate_output: bytes, - observations_input_files: MipDNAObservationsInputFiles, + mip_dna_observations_input_files: MipDNAObservationsInputFiles, ): """Test find matching profiles in Loqusdb.""" @@ -157,7 +157,7 @@ def test_get_duplicate( # WHEN retrieving the duplicated entry duplicate: dict = loqusdb_api.get_duplicate( - profile_vcf_path=observations_input_files.profile_vcf_path, + profile_vcf_path=mip_dna_observations_input_files.profile_vcf_path, profile_threshold=MipDNALoadParameters.PROFILE_THRESHOLD.value, ) @@ -167,7 +167,7 @@ def test_get_duplicate( def test_get_duplicate_non_existing( loqusdb_api: LoqusdbAPI, - observations_input_files: MipDNAObservationsInputFiles, + mip_dna_observations_input_files: MipDNAObservationsInputFiles, caplog: LogCaptureFixture, ): """Test when there are no duplicates in Loqusdb.""" @@ -177,14 +177,14 @@ def test_get_duplicate_non_existing( # WHEN extracting the duplicate duplicate: dict = loqusdb_api.get_duplicate( - profile_vcf_path=observations_input_files.profile_vcf_path, + profile_vcf_path=mip_dna_observations_input_files.profile_vcf_path, profile_threshold=MipDNALoadParameters.PROFILE_THRESHOLD.value, ) # THEN the duplicate should be empty assert not duplicate assert ( - f"No duplicates found for profile: {observations_input_files.profile_vcf_path}" + f"No duplicates found for profile: {mip_dna_observations_input_files.profile_vcf_path}" in caplog.text ) @@ -229,7 +229,7 @@ def test_delete_case_non_existing( def test_get_nr_of_variants_in_file( - loqusdb_api: LoqusdbAPI, loqusdb_load_stderr: bytes, nr_of_loaded_variants: int + loqusdb_api: LoqusdbAPI, loqusdb_load_stderr: bytes, number_of_loaded_variants: int ): """Test getting the number of variants from a Loqusdb uploaded file.""" @@ -240,7 +240,7 @@ def test_get_nr_of_variants_in_file( output = loqusdb_api.get_nr_of_variants_in_file() # THEN assert that the number of retrieved variants is correctly retrieved - assert output["variants"] == nr_of_loaded_variants + assert output["variants"] == number_of_loaded_variants def test_repr_string(loqusdb_api: LoqusdbAPI, loqusdb_binary_path: str, loqusdb_config_path: str): diff --git a/tests/cli/upload/test_cli_upload_observations.py b/tests/cli/upload/test_cli_upload_observations.py index e59c2fecff..e8dc642edf 100644 --- a/tests/cli/upload/test_cli_upload_observations.py +++ b/tests/cli/upload/test_cli_upload_observations.py @@ -9,15 +9,13 @@ from cg.cli.upload.observations import upload_observations_to_loqusdb from cg.cli.upload.observations.utils import ( get_observations_api, - get_observations_case, - get_observations_case_to_upload, - get_sequencing_method, + get_observations_verified_case, ) from cg.constants import EXIT_SUCCESS from cg.constants.constants import Workflow from cg.constants.sequencing import SequencingMethod from cg.constants.subject import PhenotypeStatus -from cg.exc import CaseNotFoundError, LoqusdbUploadCaseError +from cg.exc import CaseNotFoundError from cg.meta.observations.mip_dna_observations_api import MipDNAObservationsAPI from cg.models.cg_config import CGConfig from cg.store.models import Case, CaseSample, Sample @@ -26,22 +24,22 @@ def test_observations( - base_context: CGConfig, cli_runner: CliRunner, helpers: StoreHelpers, caplog: LogCaptureFixture + cg_context: CGConfig, cli_runner: CliRunner, helpers: StoreHelpers, caplog: LogCaptureFixture ): """Test upload of observations.""" caplog.set_level(logging.DEBUG) - store: Store = base_context.status_db + store: Store = cg_context.status_db # GIVEN a case ready to be uploaded to Loqusdb case: Case = helpers.add_case(store) case.customer.loqus_upload = True sample: Sample = helpers.add_sample(store, application_type=SequencingMethod.WES) - link = store.relate_sample(case=case, sample=sample, status=PhenotypeStatus.UNKNOWN) + link: CaseSample = store.relate_sample(case=case, sample=sample, status=PhenotypeStatus.UNKNOWN) store.session.add(link) # WHEN trying to do a dry run upload to Loqusdb result = cli_runner.invoke( - upload_observations_to_loqusdb, [case.internal_id, "--dry-run"], obj=base_context + upload_observations_to_loqusdb, [case.internal_id, "--dry-run"], obj=cg_context ) # THEN the execution should have been successful and stop at a dry run step @@ -49,111 +47,63 @@ def test_observations( assert "Dry run" in caplog.text -def test_get_observations_case(base_context: CGConfig, helpers: StoreHelpers): +def test_get_observations_case(cg_context: CGConfig, helpers: StoreHelpers): """Test get observations supported case.""" - store: Store = base_context.status_db + store: Store = cg_context.status_db # GIVEN an observations valid case case: Case = helpers.add_case(store) + case.customer.loqus_upload = True # WHEN retrieving a case given a specific case ID - extracted_case = get_observations_case(base_context, case.internal_id, upload=True) + extracted_case: Case = get_observations_verified_case(cg_context, case.internal_id, upload=True) # THEN the extracted case should match the stored one assert extracted_case == case -def test_get_observations_case_invalid_id(base_context: CGConfig, caplog: LogCaptureFixture): +def test_get_observations_case_invalid_id(cg_context: CGConfig, caplog: LogCaptureFixture): """Test get observations case providing an incorrect ID.""" caplog.set_level(logging.DEBUG) # WHEN retrieving a case given a specific case ID with pytest.raises(CaseNotFoundError): # THEN a CaseNotFoundError should be raised - get_observations_case(base_context, "invalid_case_id", upload=True) + get_observations_verified_case(cg_context, "invalid_case_id", upload=True) assert "Invalid case ID. Retrieving available cases for Loqusdb actions." in caplog.text -def test_get_observations_case_to_upload(base_context: CGConfig, helpers: StoreHelpers): +def test_get_observations_case_to_upload(cg_context: CGConfig, helpers: StoreHelpers): """Test get case ready to be uploaded to Loqusdb.""" - store: Store = base_context.status_db + store: Store = cg_context.status_db # GIVEN a case ready to be uploaded to Loqusdb case: Case = helpers.add_case(store) case.customer.loqus_upload = True # WHEN retrieving a case given a specific case ID - extracted_case = get_observations_case_to_upload(base_context, case.internal_id) + extracted_case: Case = get_observations_verified_case(cg_context, case.internal_id, upload=True) # THEN the extracted case should match the stored one assert extracted_case == case -def test_get_observations_api(base_context: CGConfig, helpers: StoreHelpers): +def test_get_observations_api(cg_context: CGConfig, helpers: StoreHelpers): """Test get observation API given a Loqusdb supported case.""" - store: Store = base_context.status_db + store: Store = cg_context.status_db # GIVEN a Loqusdb supported case case: Case = helpers.add_case(store, data_analysis=Workflow.MIP_DNA) sample: Sample = helpers.add_sample(store, application_type=SequencingMethod.WES) - link = store.relate_sample(case=case, sample=sample, status=PhenotypeStatus.UNKNOWN) + link: CaseSample = store.relate_sample(case=case, sample=sample, status=PhenotypeStatus.UNKNOWN) store.session.add(link) # WHEN retrieving the observation API - observations_api: MipDNAObservationsAPI = get_observations_api(base_context, case) + observations_api: MipDNAObservationsAPI = get_observations_api( + context=cg_context, case_id=case.internal_id, upload=True + ) # THEN a MIP-DNA API should be returned assert observations_api assert isinstance(observations_api, MipDNAObservationsAPI) - - -def test_get_sequencing_method(base_context: CGConfig, helpers: StoreHelpers): - """Test sequencing method extraction for Loqusdb upload.""" - store: Store = base_context.status_db - - # GIVEN a case object with a WGS sequencing method - case: Case = helpers.add_case(store) - sample: Sample = helpers.add_sample(store, application_type=SequencingMethod.WGS) - link = store.relate_sample(case=case, sample=sample, status=PhenotypeStatus.UNKNOWN) - store.session.add(link) - - # WHEN getting the sequencing method - sequencing_method: SequencingMethod = get_sequencing_method(case) - - # THEN the obtained sequencing method should be WGS - assert sequencing_method == SequencingMethod.WGS - - -def test_get_sequencing_method_exception( - base_context: CGConfig, - helpers: StoreHelpers, - wgs_application_tag: str, - external_wes_application_tag: str, - caplog: LogCaptureFixture, -): - """Test sequencing method extraction for Loqusdb upload when a case contains multiple sequencing types.""" - store: Store = base_context.status_db - - # GIVEN a case object with a WGS and WES mixed sequencing methods - case: Case = helpers.add_case(store) - sample_wgs: Sample = helpers.add_sample( - store, application_tag=wgs_application_tag, application_type=SequencingMethod.WGS - ) - sample_wes: Sample = helpers.add_sample( - store, application_tag=external_wes_application_tag, application_type=SequencingMethod.WES - ) - link_1: CaseSample = store.relate_sample( - case=case, sample=sample_wgs, status=PhenotypeStatus.UNKNOWN - ) - link_2: CaseSample = store.relate_sample( - case=case, sample=sample_wes, status=PhenotypeStatus.UNKNOWN - ) - store.session.add_all([link_1, link_2]) - - # WHEN getting the sequencing method - with pytest.raises(LoqusdbUploadCaseError): - # THEN a LoqusdbUploadCaseError should be raised - get_sequencing_method(case) - - assert f"Case {case.internal_id} has a mixed analysis type. Cancelling action." in caplog.text diff --git a/tests/conftest.py b/tests/conftest.py index 0572ff32cf..e12ed3f5fc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,8 +27,13 @@ from cg.apps.lims import LimsAPI from cg.apps.slurm.slurm_api import SlurmAPI from cg.constants import FileExtensions, SequencingFileTag, Workflow -from cg.constants.constants import CaseActions, CustomerId, FileFormat, GenomeVersion, Strandedness -from cg.constants.demultiplexing import DemultiplexingDirsAndFiles +from cg.constants.constants import ( + CaseActions, + CustomerId, + FileFormat, + GenomeVersion, + Strandedness, +) from cg.constants.gene_panel import GenePanelMasterList from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG from cg.constants.priority import SlurmQos @@ -53,7 +58,10 @@ from cg.models.flow_cell.flow_cell import FlowCellDirectoryData from cg.models.raredisease.raredisease import RarediseaseSampleSheetHeaders from cg.models.rnafusion.rnafusion import RnafusionParameters, RnafusionSampleSheetEntry -from cg.models.taxprofiler.taxprofiler import TaxprofilerParameters, TaxprofilerSampleSheetEntry +from cg.models.taxprofiler.taxprofiler import ( + TaxprofilerParameters, + TaxprofilerSampleSheetEntry, +) from cg.models.tomte.tomte import TomteParameters, TomteSampleSheetHeaders from cg.store.database import create_all_tables, drop_all_tables, initialize_database from cg.store.models import Bed, BedVersion, Case, Customer, Order, Organism, Sample @@ -87,8 +95,13 @@ "tests.fixture_plugins.delivery_fixtures.path_fixtures", "tests.fixture_plugins.quality_controller_fixtures.sequencing_qc_fixtures", "tests.fixture_plugins.quality_controller_fixtures.sequencing_qc_check_scenario", + "tests.fixture_plugins.loqusdb_fixtures.loqusdb_api_fixtures", + "tests.fixture_plugins.loqusdb_fixtures.loqusdb_output_fixtures", + "tests.fixture_plugins.observations_fixtures.observations_api_fixtures", + "tests.fixture_plugins.observations_fixtures.observations_input_files_fixtures", ] + # Case fixtures @@ -348,22 +361,6 @@ def base_config_dict() -> dict: "sender_email": "test@gmail.com", "sender_password": "", }, - "loqusdb": { - "binary_path": "binary", - "config_path": "config", - }, - "loqusdb-wes": { - "binary_path": "binary_wes", - "config_path": "config_wes", - }, - "loqusdb-somatic": { - "binary_path": "binary_somatic", - "config_path": "config_somatic", - }, - "loqusdb-tumor": { - "binary_path": "binary_tumor", - "config_path": "config_tumor", - }, } @@ -1280,39 +1277,39 @@ def updated_store_with_demultiplexed_samples( return store -@pytest.fixture(name="collaboration_id") +@pytest.fixture def collaboration_id() -> str: """Return a default customer group.""" return "hospital_collaboration" -@pytest.fixture(name="customer_rare_diseases") -def customer_rare_diseases(collaboration_id: str, customer_id: str) -> Customer: +@pytest.fixture +def mip_dna_customer(collaboration_id: str, customer_id: str) -> Customer: """Return a Rare Disease customer.""" return Customer( - name="CMMS", - internal_id="cust003", + name="Klinisk Immunologi", + internal_id=CustomerId.CUST004, loqus_upload=True, ) -@pytest.fixture(name="customer_balsamic") -def customer_balsamic(collaboration_id: str, customer_id: str) -> Customer: +@pytest.fixture +def balsamic_customer(collaboration_id: str, customer_id: str) -> Customer: """Return a Cancer customer.""" return Customer( name="AML", - internal_id="cust110", + internal_id=CustomerId.CUST110, loqus_upload=True, ) -@pytest.fixture(name="external_wes_application_tag") +@pytest.fixture def external_wes_application_tag() -> str: """Return the external whole exome sequencing application tag.""" return "EXXCUSR000" -@pytest.fixture(name="wgs_application_tag") +@pytest.fixture def wgs_application_tag() -> str: """Return the WGS application tag.""" return "WGSPCFC030" @@ -1340,43 +1337,43 @@ def store() -> Generator[Store, None, None]: drop_all_tables() -@pytest.fixture(name="apptag_rna") +@pytest.fixture def apptag_rna() -> str: """Return the RNA application tag.""" return "RNAPOAR025" -@pytest.fixture(name="bed_name") +@pytest.fixture def bed_name() -> str: """Return a bed model name attribute.""" return "Bed" -@pytest.fixture(name="bed_version_file_name") -def bed_version_filename(bed_name: str) -> str: +@pytest.fixture +def bed_version_file_name(bed_name: str) -> str: """Return a bed version model file name attribute.""" return f"{bed_name}.bed" -@pytest.fixture(name="bed_version_short_name") +@pytest.fixture def bed_version_short_name() -> str: """Return a bed version model short name attribute.""" return "bed_short_name_0.0" -@pytest.fixture(name="invoice_address") +@pytest.fixture def invoice_address() -> str: """Return an invoice address.""" return "Test street" -@pytest.fixture(name="invoice_reference") +@pytest.fixture def invoice_reference() -> str: """Return an invoice reference.""" return "ABCDEF" -@pytest.fixture(name="prices") +@pytest.fixture def prices() -> dict[str, int]: """Return dictionary with prices for each priority status.""" return {"standard": 10, "priority": 20, "express": 30, "research": 5} @@ -1759,12 +1756,6 @@ def hk_uri() -> str: return "sqlite:///" -@pytest.fixture(name="loqusdb_id") -def loqusdb_id() -> str: - """Returns a Loqusdb mock ID.""" - return "01ab23cd" - - @pytest.fixture(name="context_config") def context_config( cg_uri: str, @@ -1925,10 +1916,13 @@ def context_config( "password": "password", "username": "user", }, - "loqusdb": {"binary_path": "loqusdb", "config_path": "loqusdb-stage.yaml"}, - "loqusdb-wes": {"binary_path": "loqusdb", "config_path": "loqusdb-wes-stage.yaml"}, - "loqusdb-somatic": {"binary_path": "loqusdb", "config_path": "loqusdb-somatic-stage.yaml"}, - "loqusdb-tumor": {"binary_path": "loqusdb", "config_path": "loqusdb-tumor-stage.yaml"}, + "loqusdb": {"binary_path": "loqusdb", "config_path": "loqusdb.yaml"}, + "loqusdb-wes": {"binary_path": "loqusdb-wes", "config_path": "loqusdb-wes.yaml"}, + "loqusdb-somatic": { + "binary_path": "loqusdb-somatic", + "config_path": "loqusdb-somatic.yaml", + }, + "loqusdb-tumor": {"binary_path": "loqusdb-tumor", "config_path": "loqusdb-tumor.yaml"}, "microsalt": { "binary_path": "echo", "conda_binary": "a_conda_binary", @@ -2448,73 +2442,6 @@ def raredisease_deliverables_file_path(raredisease_dir, raredisease_case_id) -> ).with_suffix(FileExtensions.YAML) -@pytest.fixture(scope="function") -def raredisease_context( - cg_context: CGConfig, - helpers: StoreHelpers, - nf_analysis_housekeeper: HousekeeperAPI, - trailblazer_api: MockTB, - raredisease_case_id: str, - sample_id: str, - no_sample_case_id: str, - total_sequenced_reads_pass: int, - apptag_rna: str, - case_id_not_enough_reads: str, - sample_id_not_enough_reads: str, - total_sequenced_reads_not_pass: int, -) -> CGConfig: - """context to use in cli""" - cg_context.housekeeper_api_ = nf_analysis_housekeeper - cg_context.trailblazer_api_ = trailblazer_api - cg_context.meta_apis["analysis_api"] = RarediseaseAnalysisAPI(config=cg_context) - status_db: Store = cg_context.status_db - - # Create ERROR case with NO SAMPLES - helpers.add_case(status_db, internal_id=no_sample_case_id, name=no_sample_case_id) - - # Create textbook case with enough reads - case_enough_reads: Case = helpers.add_case( - store=status_db, - internal_id=raredisease_case_id, - name=raredisease_case_id, - data_analysis=Workflow.RAREDISEASE, - ) - - sample_raredisease_case_enough_reads: Sample = helpers.add_sample( - status_db, - internal_id=sample_id, - last_sequenced_at=datetime.now(), - reads=total_sequenced_reads_pass, - application_tag=apptag_rna, - ) - - helpers.add_relationship( - status_db, - case=case_enough_reads, - sample=sample_raredisease_case_enough_reads, - ) - - # Create case without enough reads - case_not_enough_reads: Case = helpers.add_case( - store=status_db, - internal_id=case_id_not_enough_reads, - name=case_id_not_enough_reads, - data_analysis=Workflow.RAREDISEASE, - ) - - sample_not_enough_reads: Sample = helpers.add_sample( - status_db, - internal_id=sample_id_not_enough_reads, - last_sequenced_at=datetime.now(), - reads=total_sequenced_reads_not_pass, - application_tag=apptag_rna, - ) - - helpers.add_relationship(status_db, case=case_not_enough_reads, sample=sample_not_enough_reads) - - return cg_context - - @pytest.fixture(scope="function") def deliverable_data(raredisease_dir: Path, raredisease_case_id: str, sample_id: str) -> dict: return { @@ -3427,14 +3354,6 @@ def taxprofiler_params_file_path(taxprofiler_dir, taxprofiler_case_id) -> Path: ).with_suffix(FileExtensions.YAML) -@pytest.fixture(scope="function") -def taxprofiler_nexflow_config_file_path(taxprofiler_dir, taxprofiler_case_id) -> Path: - """Path to config file.""" - return Path( - taxprofiler_dir, taxprofiler_case_id, f"{taxprofiler_case_id}_nextflow_config" - ).with_suffix(FileExtensions.JSON) - - @pytest.fixture(scope="function") def taxprofiler_hermes_deliverables( taxprofiler_deliverable_data: dict, taxprofiler_case_id: str @@ -3688,25 +3607,6 @@ def taxprofiler_mock_config(taxprofiler_dir: Path, taxprofiler_case_id: str) -> ).touch(exist_ok=True) -@pytest.fixture(scope="function") -def taxprofiler_deliverable_data( - taxprofiler_dir: Path, taxprofiler_case_id: str, sample_id: str -) -> dict: - return { - "files": [ - { - "path": f"{taxprofiler_dir}/{taxprofiler_case_id}/multiqc/multiqc_report.html", - "path_index": "", - "step": "report", - "tag": ["multiqc-html"], - "id": taxprofiler_case_id, - "format": "html", - "mandatory": True, - }, - ] - } - - @pytest.fixture(scope="function") def taxprofiler_deliverables_response_data( create_multiqc_html_file, diff --git a/tests/fixture_plugins/loqusdb_fixtures/loqusdb_api_fixtures.py b/tests/fixture_plugins/loqusdb_fixtures/loqusdb_api_fixtures.py new file mode 100644 index 0000000000..55084498a1 --- /dev/null +++ b/tests/fixture_plugins/loqusdb_fixtures/loqusdb_api_fixtures.py @@ -0,0 +1,57 @@ +"""Loqusdb API fixtures.""" + +import pytest + +from cg.apps.loqus import LoqusdbAPI +from cg.models.cg_config import CGConfig +from tests.mocks.process_mock import ProcessMock + + +@pytest.fixture +def loqusdb_id() -> str: + """Returns a Loqusdb mock ID.""" + return "01ab23cd" + + +@pytest.fixture +def loqusdb_binary_path(cg_context: CGConfig) -> str: + """Return Loqusdb binary path.""" + return cg_context.loqusdb.binary_path + + +@pytest.fixture +def loqusdb_config_path(cg_context: CGConfig) -> str: + """Return Loqusdb config dictionary.""" + return cg_context.loqusdb.config_path + + +@pytest.fixture +def loqusdb_process(loqusdb_binary_path: str, loqusdb_config_path: str) -> ProcessMock: + """Return mocked process instance.""" + return ProcessMock(binary=loqusdb_binary_path, config=loqusdb_config_path) + + +@pytest.fixture +def loqusdb_process_exception(loqusdb_binary_path: str, loqusdb_config_path: str) -> ProcessMock: + """Return error process instance.""" + return ProcessMock(binary=loqusdb_binary_path, config=loqusdb_config_path, error=True) + + +@pytest.fixture +def loqusdb_api( + loqusdb_binary_path: str, loqusdb_config_path: str, loqusdb_process: ProcessMock +) -> LoqusdbAPI: + """Return Loqusdb API.""" + loqusdb_api = LoqusdbAPI(binary_path=loqusdb_binary_path, config_path=loqusdb_config_path) + loqusdb_api.process = loqusdb_process + return loqusdb_api + + +@pytest.fixture +def loqusdb_api_exception( + loqusdb_binary_path: str, loqusdb_config_path: str, loqusdb_process_exception: ProcessMock +) -> LoqusdbAPI: + """Return Loqusdb API with mocked error process.""" + loqusdb_api = LoqusdbAPI(binary_path=loqusdb_binary_path, config_path=loqusdb_config_path) + loqusdb_api.process = loqusdb_process_exception + return loqusdb_api diff --git a/tests/apps/loqus/conftest.py b/tests/fixture_plugins/loqusdb_fixtures/loqusdb_output_fixtures.py similarity index 64% rename from tests/apps/loqus/conftest.py rename to tests/fixture_plugins/loqusdb_fixtures/loqusdb_output_fixtures.py index 102740adc9..81ce2a97c2 100644 --- a/tests/apps/loqus/conftest.py +++ b/tests/fixture_plugins/loqusdb_fixtures/loqusdb_output_fixtures.py @@ -1,20 +1,11 @@ -""" Conftest for Loqusdb API.""" +"""Loqusdb upload output fixtures.""" import pytest -from cg.apps.loqus import LoqusdbAPI -from cg.constants.observations import LoqusdbInstance -from cg.models.cg_config import CGConfig, CommonAppConfig -from tests.mocks.process_mock import ProcessMock -from tests.models.observations.conftest import ( - observations_input_files, - observations_input_files_raw, -) - LOQUSDB_OUTPUT = ( b"2018-11-29 08:41:38 130-229-8-20-dhcp.local " b"mongo_adapter.client[77135] INFO Connecting to " - b"uri:mongodb://None:None@localhost:27017\n" + b"uri:mongodb://None@localhost:27017\n" b"2018-11-29 08:41:38 130-229-8-20-dhcp.local " b"mongo_adapter.client[77135] INFO Connection " b"established\n2018-11-29 08:41:38 130-229-8-20-dhcp.local " @@ -82,7 +73,7 @@ ) LOQUSDB_DELETE_STDERR = b"""2022-09-22 12:30:07 username loqusdb.commands.cli[20689] INFO Running loqusdb version 2.6.9 -2022-09-22 12:30:07 username mongo_adapter.client[20689] INFO Connecting to uri:mongodb://None:None@localhost:27017 +2022-09-22 12:30:07 username mongo_adapter.client[20689] INFO Connecting to uri:mongodb://None@localhost:27017 2022-09-22 12:30:07 username mongo_adapter.client[20689] INFO Connection established 2022-09-22 12:30:07 username mongo_adapter.adapter[20689] INFO Use database loqusdb 2022-09-22 12:30:07 username loqusdb.plugins.mongo.case[20689] INFO Removing case yellowhog from database @@ -90,81 +81,12 @@ 2022-09-22 12:30:07 username loqusdb.utils.delete[20689] INFO Start deleting chromosome 1""" LOQUSDB_DELETE_NONEXISTING_STDERR = b"""2022-09-22 11:40:04 username loqusdb.commands.cli[19944] INFO Running loqusdb version 2.6.9 -2022-09-22 11:40:04 username mongo_adapter.client[19944] INFO Connecting to uri:mongodb://None:None@localhost:27017 +2022-09-22 11:40:04 username mongo_adapter.client[19944] INFO Connecting to uri:mongodb://None@localhost:27017 2022-09-22 11:40:04 username mongo_adapter.client[19944] INFO Connection established 2022-09-22 11:40:04 username mongo_adapter.adapter[19944] INFO Use database loqusdb 2022-09-22 11:40:04 username loqusdb.commands.delete[19944] WARNING Case yellowhog does not exist in database""" -@pytest.fixture -def loqusdb_config_dict() -> dict[LoqusdbInstance, dict]: - """Return Loqusdb config dictionary.""" - return { - LoqusdbInstance.WGS: {"binary_path": "binary", "config_path": "config"}, - LoqusdbInstance.WES: {"binary_path": "binary_wes", "config_path": "config_wes"}, - LoqusdbInstance.SOMATIC: {"binary_path": "binary_somatic", "config_path": "config_somatic"}, - LoqusdbInstance.TUMOR: {"binary_path": "binary_tumor", "config_path": "config_tumor"}, - } - - -@pytest.fixture -def cg_config_locusdb( - loqusdb_config_dict: dict[LoqusdbInstance, dict], cg_config_object: CGConfig -) -> CGConfig: - """Return CG config for Loqusdb.""" - cg_config_object.loqusdb = CommonAppConfig(**loqusdb_config_dict[LoqusdbInstance.WGS]) - cg_config_object.loqusdb_wes = CommonAppConfig(**loqusdb_config_dict[LoqusdbInstance.WES]) - cg_config_object.loqusdb_somatic = CommonAppConfig( - **loqusdb_config_dict[LoqusdbInstance.SOMATIC] - ) - cg_config_object.loqusdb_tumor = CommonAppConfig(**loqusdb_config_dict[LoqusdbInstance.TUMOR]) - return cg_config_object - - -@pytest.fixture -def loqusdb_binary_path(loqusdb_config_dict: dict[LoqusdbInstance, dict]) -> str: - """Return Loqusdb binary path.""" - return loqusdb_config_dict[LoqusdbInstance.WGS]["binary_path"] - - -@pytest.fixture -def loqusdb_config_path(loqusdb_config_dict: dict[LoqusdbInstance, dict]) -> str: - """Return Loqusdb config dictionary.""" - return loqusdb_config_dict[LoqusdbInstance.WGS]["config_path"] - - -@pytest.fixture -def loqusdb_process(loqusdb_binary_path: str, loqusdb_config_path: str) -> ProcessMock: - """Return mocked process instance.""" - return ProcessMock(binary=loqusdb_binary_path, config=loqusdb_config_path) - - -@pytest.fixture -def loqusdb_process_exception(loqusdb_binary_path: str, loqusdb_config_path: str) -> ProcessMock: - """Return error process instance.""" - return ProcessMock(binary=loqusdb_binary_path, config=loqusdb_config_path, error=True) - - -@pytest.fixture -def loqusdb_api( - loqusdb_binary_path: str, loqusdb_config_path: str, loqusdb_process: ProcessMock -) -> LoqusdbAPI: - """Return Loqusdb API.""" - loqusdb_api = LoqusdbAPI(binary_path=loqusdb_binary_path, config_path=loqusdb_config_path) - loqusdb_api.process = loqusdb_process - return loqusdb_api - - -@pytest.fixture -def loqusdb_api_exception( - loqusdb_binary_path: str, loqusdb_config_path: str, loqusdb_process_exception: ProcessMock -) -> LoqusdbAPI: - """Return Loqusdb API with mocked error process.""" - loqusdb_api = LoqusdbAPI(binary_path=loqusdb_binary_path, config_path=loqusdb_config_path) - loqusdb_api.process = loqusdb_process_exception - return loqusdb_api - - @pytest.fixture def loqusdb_load_stderr() -> bytes: """Return Loqusdb stderr for a successful load.""" @@ -196,6 +118,6 @@ def loqusdb_delete_non_existing_stderr() -> bytes: @pytest.fixture -def nr_of_loaded_variants() -> int: +def number_of_loaded_variants() -> int: """Return number of loaded variants.""" return 15 diff --git a/tests/fixture_plugins/observations_fixtures/__init__.py b/tests/fixture_plugins/observations_fixtures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fixture_plugins/observations_fixtures/observations_api_fixtures.py b/tests/fixture_plugins/observations_fixtures/observations_api_fixtures.py new file mode 100644 index 0000000000..14e4c4f5da --- /dev/null +++ b/tests/fixture_plugins/observations_fixtures/observations_api_fixtures.py @@ -0,0 +1,82 @@ +"""Loqusdb API fixtures.""" + +import pytest +from pytest_mock import MockFixture + +from cg.apps.lims import LimsAPI +from cg.apps.loqus import LoqusdbAPI +from cg.constants.observations import LOQUSDB_ID +from cg.constants.sample_sources import SourceType +from cg.meta.observations.balsamic_observations_api import BalsamicObservationsAPI +from cg.meta.observations.mip_dna_observations_api import MipDNAObservationsAPI +from cg.models.cg_config import CGConfig +from cg.store.models import Case, Customer +from cg.store.store import Store + + +@pytest.fixture +def balsamic_observations_api( + cg_context: CGConfig, + analysis_store: Store, + lims_api: LimsAPI, + loqusdb_api: LoqusdbAPI, + case_id: str, + balsamic_customer: Customer, + number_of_loaded_variants: int, + loqusdb_id: str, + mocker: MockFixture, +) -> BalsamicObservationsAPI: + """Cancer observations API fixture.""" + balsamic_observations_api: BalsamicObservationsAPI = BalsamicObservationsAPI(cg_context) + balsamic_observations_api.store = analysis_store + balsamic_observations_api.loqusdb_somatic_api = loqusdb_api + balsamic_observations_api.loqusdb_tumor_api = loqusdb_api + + # Mocked case scenario for Balsamic uploads + case: Case = analysis_store.get_case_by_internal_id(case_id) + case.customer.internal_id = balsamic_customer.internal_id + case.samples[0].is_tumour = True + + # Mocked Loqusdb API scenario for Balsamic uploads + mocker.patch.object(LoqusdbAPI, "load", return_value={"variants": number_of_loaded_variants}) + mocker.patch.object( + LoqusdbAPI, "get_case", return_value={"case_id": case_id, LOQUSDB_ID: loqusdb_id} + ) + + # Mocked LIMS API scenario + mocker.patch.object(LimsAPI, "get_source", return_value=SourceType.TISSUE) + + return balsamic_observations_api + + +@pytest.fixture +def mip_dna_observations_api( + cg_context: CGConfig, + analysis_store: Store, + lims_api: LimsAPI, + loqusdb_api: LoqusdbAPI, + case_id: str, + mip_dna_customer: Customer, + number_of_loaded_variants: int, + loqusdb_id: str, + mocker: MockFixture, +) -> MipDNAObservationsAPI: + """Rare diseases observations API fixture.""" + mip_dna_observations_api: MipDNAObservationsAPI = MipDNAObservationsAPI(cg_context) + mip_dna_observations_api.store = analysis_store + mip_dna_observations_api.loqusdb_api = loqusdb_api + + # Mocked case scenario for MIP-DNA uploads + case: Case = analysis_store.get_case_by_internal_id(case_id) + case.customer.internal_id = mip_dna_customer.internal_id + + # Mocked Loqusdb API scenario for MIP-DNA uploads + mocker.patch.object(LoqusdbAPI, "load", return_value={"variants": number_of_loaded_variants}) + mocker.patch.object( + LoqusdbAPI, "get_case", return_value={"case_id": case_id, LOQUSDB_ID: loqusdb_id} + ) + + # Mocked LIMS API scenario + mocker.patch.object(LimsAPI, "get_source", return_value=SourceType.TISSUE) + + return mip_dna_observations_api diff --git a/tests/models/observations/conftest.py b/tests/fixture_plugins/observations_fixtures/observations_input_files_fixtures.py similarity index 65% rename from tests/models/observations/conftest.py rename to tests/fixture_plugins/observations_fixtures/observations_input_files_fixtures.py index 73f55e7d14..edc044fc2e 100644 --- a/tests/models/observations/conftest.py +++ b/tests/fixture_plugins/observations_fixtures/observations_input_files_fixtures.py @@ -1,4 +1,4 @@ -"""ObservationsInputFiles test fixtures.""" +"""Loqusdb input files fixtures.""" from pathlib import Path @@ -10,27 +10,8 @@ ) -@pytest.fixture(name="observations_input_files_raw") -def observations_input_files_raw(case_id: str, filled_file: Path) -> dict: - """Return raw observations input files for rare diseases.""" - return { - "family_ped_path": filled_file, - "profile_vcf_path": filled_file, - "snv_vcf_path": filled_file, - "sv_vcf_path": None, - } - - -@pytest.fixture(name="observations_input_files") -def observations_input_files( - observations_input_files_raw: dict, -) -> MipDNAObservationsInputFiles: - """Return raw observations input files for rare diseases WES analysis.""" - return MipDNAObservationsInputFiles(**observations_input_files_raw) - - -@pytest.fixture(name="balsamic_observations_input_files_raw") -def balsamic_observations_input_files_raw(case_id: str, filled_file: Path) -> dict: +@pytest.fixture +def balsamic_observations_input_files_raw(case_id: str, filled_file: Path) -> dict[str, Path]: """Return raw observations input files for cancer.""" return { "snv_germline_vcf_path": filled_file, @@ -40,9 +21,28 @@ def balsamic_observations_input_files_raw(case_id: str, filled_file: Path) -> di } -@pytest.fixture(name="balsamic_observations_input_files") +@pytest.fixture def balsamic_observations_input_files( - balsamic_observations_input_files_raw: dict, + balsamic_observations_input_files_raw: dict[str, Path], ) -> BalsamicObservationsInputFiles: """Return raw observations input files for cancer WGS analysis.""" return BalsamicObservationsInputFiles(**balsamic_observations_input_files_raw) + + +@pytest.fixture +def mip_dna_observations_input_files_raw(case_id: str, filled_file: Path) -> dict[str, Path]: + """Return raw observations input files for rare diseases.""" + return { + "family_ped_path": filled_file, + "profile_vcf_path": filled_file, + "snv_vcf_path": filled_file, + "sv_vcf_path": None, + } + + +@pytest.fixture +def mip_dna_observations_input_files( + mip_dna_observations_input_files_raw: dict[str, Path], +) -> MipDNAObservationsInputFiles: + """Return raw observations input files for rare diseases WES analysis.""" + return MipDNAObservationsInputFiles(**mip_dna_observations_input_files_raw) diff --git a/tests/meta/observations/conftest.py b/tests/meta/observations/conftest.py deleted file mode 100644 index 85853683e6..0000000000 --- a/tests/meta/observations/conftest.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Fixtures for observations.""" - -import pytest - -from cg.apps.loqus import LoqusdbAPI -from cg.constants.observations import LOQUSDB_ID -from cg.constants.sequencing import SequencingMethod -from cg.meta.observations.balsamic_observations_api import BalsamicObservationsAPI -from cg.meta.observations.mip_dna_observations_api import MipDNAObservationsAPI -from cg.models.cg_config import CGConfig -from cg.store.store import Store -from tests.apps.loqus.conftest import ( - loqusdb_api, - loqusdb_binary_path, - loqusdb_config_dict, - loqusdb_config_path, - loqusdb_process, - nr_of_loaded_variants, -) -from tests.cli.conftest import base_context -from tests.models.observations.conftest import ( - balsamic_observations_input_files, - balsamic_observations_input_files_raw, - observations_input_files, - observations_input_files_raw, -) - - -class MockLoqusdbAPI(LoqusdbAPI): - """Mock LoqusdbAPI class.""" - - def __init__(self, binary_path: str, config_path: str): - super().__init__(binary_path, config_path) - - @staticmethod - def load(*args, **kwargs) -> dict: - """Mock load method.""" - _ = args - _ = kwargs - return dict(variants=15) - - @staticmethod - def get_case(*args, **kwargs) -> dict | None: - """Mock get_case method.""" - _ = args - _ = kwargs - return {"case_id": "case_id", LOQUSDB_ID: "123"} - - @staticmethod - def get_duplicate(*args, **kwargs) -> dict | None: - """Mock get_duplicate method.""" - _ = args - _ = kwargs - return {"case_id": "case_id"} - - @staticmethod - def delete_case(*args, **kwargs) -> None: - """Mock delete_case method.""" - _ = args - _ = kwargs - return None - - -@pytest.fixture(name="mock_loqusdb_api") -def mock_loqusdb_api(filled_file) -> MockLoqusdbAPI: - """Mock Loqusdb API.""" - return MockLoqusdbAPI(binary_path=filled_file, config_path=filled_file) - - -@pytest.fixture(name="mip_dna_observations_api") -def mip_dna_observations_api( - cg_config_object: CGConfig, mock_loqusdb_api: MockLoqusdbAPI, analysis_store: Store -) -> MipDNAObservationsAPI: - """Rare diseases observations API fixture.""" - mip_dna_observations_api: MipDNAObservationsAPI = MipDNAObservationsAPI( - config=cg_config_object, sequencing_method=SequencingMethod.WGS - ) - mip_dna_observations_api.store = analysis_store - mip_dna_observations_api.loqusdb_api = mock_loqusdb_api - return mip_dna_observations_api - - -@pytest.fixture(name="balsamic_observations_api") -def balsamic_observations_api( - cg_config_object: CGConfig, mock_loqusdb_api: MockLoqusdbAPI, analysis_store: Store -) -> BalsamicObservationsAPI: - """Rare diseases observations API fixture.""" - balsamic_observations_api: BalsamicObservationsAPI = BalsamicObservationsAPI( - config=cg_config_object, sequencing_method=SequencingMethod.WGS - ) - balsamic_observations_api.store = analysis_store - balsamic_observations_api.loqusdb_somatic_api = mock_loqusdb_api - balsamic_observations_api.loqusdb_tumor_api = mock_loqusdb_api - return balsamic_observations_api diff --git a/tests/meta/observations/test_balsamic_observations_api.py b/tests/meta/observations/test_balsamic_observations_api.py new file mode 100644 index 0000000000..131a6fece5 --- /dev/null +++ b/tests/meta/observations/test_balsamic_observations_api.py @@ -0,0 +1,190 @@ +"""Test Balsamic observations API.""" + +import logging + +import pytest +from _pytest.logging import LogCaptureFixture +from pytest_mock import MockFixture + +from cg.constants.constants import CancerAnalysisType +from cg.exc import LoqusdbDuplicateRecordError +from cg.meta.observations.balsamic_observations_api import BalsamicObservationsAPI +from cg.meta.workflow.balsamic import BalsamicAnalysisAPI +from cg.models.observations.input_files import BalsamicObservationsInputFiles +from cg.store.models import Case + + +def test_is_analysis_type_eligible_for_observations_upload( + case_id: str, balsamic_observations_api: BalsamicObservationsAPI, mocker: MockFixture +): + """Test if the analysis type is eligible for observation uploads.""" + + # GIVEN a case ID and a Balsamic observations API + + # GIVEN a case with tumor samples + mocker.patch.object(BalsamicAnalysisAPI, "is_analysis_normal_only", return_value=False) + + # WHEN checking analysis type eligibility for a case + is_analysis_type_eligible_for_observations_upload: bool = ( + balsamic_observations_api.is_analysis_type_eligible_for_observations_upload(case_id) + ) + + # THEN the analysis type should be eligible for observation uploads + assert is_analysis_type_eligible_for_observations_upload + + +def test_is_analysis_type_not_eligible_for_observations_upload( + case_id: str, + balsamic_observations_api: BalsamicObservationsAPI, + mocker: MockFixture, + caplog: LogCaptureFixture, +): + """Test if the analysis type is not eligible for observation uploads.""" + + # GIVEN a case ID and a Balsamic observations API + + # GIVEN a case without tumor samples (normal-only analysis) + mocker.patch.object(BalsamicAnalysisAPI, "is_analysis_normal_only", return_value=True) + + # WHEN checking analysis type eligibility for a case + is_analysis_type_eligible_for_observations_upload: bool = ( + balsamic_observations_api.is_analysis_type_eligible_for_observations_upload(case_id) + ) + + # THEN the analysis type should not be eligible for observation uploads + assert not is_analysis_type_eligible_for_observations_upload + assert f"Normal only analysis {case_id} is not supported for Loqusdb uploads" in caplog.text + + +def test_is_case_eligible_for_observations_upload( + case_id: str, balsamic_observations_api: BalsamicObservationsAPI, mocker: MockFixture +): + """Test whether a case is eligible for Balsamic observation uploads.""" + + # GIVEN a case and a Balsamic observations API + case: Case = balsamic_observations_api.analysis_api.status_db.get_case_by_internal_id(case_id) + + # GIVEN an eligible sequencing method and a case with tumor samples + mocker.patch.object( + BalsamicAnalysisAPI, "get_data_analysis_type", return_value=CancerAnalysisType.TUMOR_WGS + ) + mocker.patch.object(BalsamicAnalysisAPI, "is_analysis_normal_only", return_value=False) + + # WHEN checking the upload eligibility for a case + is_case_eligible_for_observations_upload: bool = ( + balsamic_observations_api.is_case_eligible_for_observations_upload(case) + ) + + # THEN the case should be eligible for observation uploads + assert is_case_eligible_for_observations_upload + + +def test_is_case_not_eligible_for_observations_upload( + case_id: str, + balsamic_observations_api: BalsamicObservationsAPI, + mocker: MockFixture, + caplog: LogCaptureFixture, +): + """Test whether a case is eligible for Balsamic observation uploads.""" + + # GIVEN a case and a Balsamic observations API + case: Case = balsamic_observations_api.analysis_api.status_db.get_case_by_internal_id(case_id) + + # GIVEN a case with tumor sample and an invalid sequencing type + mocker.patch.object(BalsamicAnalysisAPI, "is_analysis_normal_only", return_value=False) + mocker.patch.object( + BalsamicAnalysisAPI, "get_data_analysis_type", return_value=CancerAnalysisType.TUMOR_PANEL + ) + + # WHEN checking the upload eligibility for a case + is_case_eligible_for_observations_upload: bool = ( + balsamic_observations_api.is_case_eligible_for_observations_upload(case) + ) + + # THEN the case should not be eligible for observation uploads + assert not is_case_eligible_for_observations_upload + + +def test_load_observations( + case_id: str, + loqusdb_id: str, + balsamic_observations_api: BalsamicObservationsAPI, + balsamic_observations_input_files: BalsamicObservationsInputFiles, + number_of_loaded_variants: int, + caplog: LogCaptureFixture, + mocker: MockFixture, +): + """Test loading of Balsamic case observations.""" + caplog.set_level(logging.DEBUG) + + # GIVEN an observations API and a list of input files for upload + case: Case = balsamic_observations_api.store.get_case_by_internal_id(case_id) + mocker.patch.object( + BalsamicObservationsAPI, + "get_observations_input_files", + return_value=balsamic_observations_input_files, + ) + + # GIVEN an observations API mocked scenario + mocker.patch.object(BalsamicObservationsAPI, "is_duplicate", return_value=False) + + # WHEN loading the case to Loqusdb + balsamic_observations_api.load_observations(case) + + # THEN the observations should be loaded successfully + assert f"Uploaded {number_of_loaded_variants} variants to Loqusdb" in caplog.text + + +def test_load_duplicated_observations( + case_id: str, + balsamic_observations_api: BalsamicObservationsAPI, + caplog: LogCaptureFixture, + mocker: MockFixture, +): + """Test raise of a duplicate exception when loading Balsamic case observations.""" + caplog.set_level(logging.DEBUG) + + # GIVEN an observations API and a Balsamic case + case: Case = balsamic_observations_api.store.get_case_by_internal_id(case_id) + + # GIVEN a duplicate case in Loqusdb + mocker.patch.object(BalsamicObservationsAPI, "is_duplicate", return_value=True) + + # WHEN loading the case to Loqusdb + with pytest.raises(LoqusdbDuplicateRecordError): + balsamic_observations_api.load_observations(case) + + # THEN the observations upload should be aborted + assert f"Case {case_id} has already been uploaded to Loqusdb" in caplog.text + + +def test_load_cancer_observations( + case_id: str, + balsamic_observations_api: BalsamicObservationsAPI, + balsamic_observations_input_files: BalsamicObservationsInputFiles, + number_of_loaded_variants: int, + mocker: MockFixture, + caplog: LogCaptureFixture, +): + """Test loading of cancer case observations for Balsamic.""" + caplog.set_level(logging.DEBUG) + + # GIVEN a Balsamic observations API, a list of input files, and a cancer case + case: Case = balsamic_observations_api.store.get_case_by_internal_id(case_id) + + # GIVEN an observations API mocked scenario + mocker.patch.object( + BalsamicAnalysisAPI, + "get_data_analysis_type", + return_value=CancerAnalysisType.TUMOR_NORMAL_WGS, + ) + + # WHEN loading the case to a somatic Loqusdb instance + balsamic_observations_api.load_cancer_observations( + case=case, + input_files=balsamic_observations_input_files, + loqusdb_api=balsamic_observations_api.loqusdb_somatic_api, + ) + + # THEN the observations should be loaded successfully + assert f"Uploaded {number_of_loaded_variants} variants to Loqusdb" in caplog.text diff --git a/tests/meta/observations/test_meta_upload_observations.py b/tests/meta/observations/test_meta_upload_observations.py deleted file mode 100644 index f6c27849e1..0000000000 --- a/tests/meta/observations/test_meta_upload_observations.py +++ /dev/null @@ -1,420 +0,0 @@ -"""Test observations API methods.""" - -import logging - -import pytest -from _pytest.logging import LogCaptureFixture - -from cg.apps.loqus import LoqusdbAPI -from cg.constants.observations import ( - LoqusdbInstance, - LoqusdbMipCustomers, - MipDNALoadParameters, -) -from cg.constants.sequencing import SequencingMethod -from cg.exc import ( - CaseNotFoundError, - LoqusdbDuplicateRecordError, - LoqusdbUploadCaseError, -) -from cg.meta.observations.balsamic_observations_api import BalsamicObservationsAPI -from cg.meta.observations.mip_dna_observations_api import MipDNAObservationsAPI -from cg.models.observations.input_files import ( - BalsamicObservationsInputFiles, - MipDNAObservationsInputFiles, -) -from cg.store.models import Case, Customer -from tests.store_helpers import StoreHelpers - - -def test_observations_upload( - case_id: str, - mip_dna_observations_api: MipDNAObservationsAPI, - observations_input_files: MipDNAObservationsInputFiles, - nr_of_loaded_variants: int, - caplog: LogCaptureFixture, - mocker, -): - """Test upload observations method.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a mocked observations API and a list of mocked observations files - case: Case = mip_dna_observations_api.store.get_case_by_internal_id(internal_id=case_id) - case.customer.internal_id = LoqusdbMipCustomers.KLINISK_IMMUNOLOGI.value - mocker.patch.object( - mip_dna_observations_api, - "get_observations_input_files", - return_value=observations_input_files, - ) - mocker.patch.object(mip_dna_observations_api, "is_duplicate", return_value=False) - - # WHEN uploading the case observations to Loqusdb - mip_dna_observations_api.upload(case) - - # THEN the case should be successfully uploaded - assert f"Uploaded {nr_of_loaded_variants} variants to Loqusdb" in caplog.text - - -def test_get_loqusdb_api( - mip_dna_observations_api: MipDNAObservationsAPI, - loqusdb_config_dict: dict[LoqusdbInstance, dict], -): - """Test Loqusdb API retrieval given a Loqusdb instance.""" - - # GIVEN the expected Loqusdb config dictionary - - # GIVEN a WES Loqusdb instance and an observations API - loqusdb_instance = LoqusdbInstance.WES - - # WHEN calling the Loqusdb API get method - loqusdb_api: LoqusdbAPI = mip_dna_observations_api.get_loqusdb_api(loqusdb_instance) - - # THEN a WES loqusdb api should be returned - assert isinstance(loqusdb_api, LoqusdbAPI) - assert loqusdb_api.binary_path == loqusdb_config_dict[LoqusdbInstance.WES]["binary_path"] - assert loqusdb_api.config_path == loqusdb_config_dict[LoqusdbInstance.WES]["config_path"] - - -def test_is_duplicate( - case_id: str, - mip_dna_observations_api: MipDNAObservationsAPI, - observations_input_files: MipDNAObservationsInputFiles, - mocker, -): - """Test duplicate extraction for a case that is not in Loqusdb.""" - - # GIVEN a Loqusdb instance with no case duplicates - case: Case = mip_dna_observations_api.store.get_case_by_internal_id(internal_id=case_id) - mocker.patch.object(mip_dna_observations_api.loqusdb_api, "get_case", return_value=None) - mocker.patch.object(mip_dna_observations_api.loqusdb_api, "get_duplicate", return_value=False) - - # WHEN checking that a case has not been uploaded to Loqusdb - is_duplicate: bool = mip_dna_observations_api.is_duplicate( - case=case, - loqusdb_api=mip_dna_observations_api.loqusdb_api, - profile_vcf_path=observations_input_files.profile_vcf_path, - profile_threshold=MipDNALoadParameters.PROFILE_THRESHOLD.value, - ) - - # THEN there should be no duplicates in Loqusdb - assert is_duplicate is False - - -def test_is_duplicate_case_output( - case_id: str, - observations_input_files: MipDNAObservationsInputFiles, - mip_dna_observations_api: MipDNAObservationsAPI, -): - """Test duplicate extraction for a case that already exists in Loqusdb.""" - - # GIVEN a Loqusdb instance with a duplicated case - case: Case = mip_dna_observations_api.store.get_case_by_internal_id(internal_id=case_id) - - # WHEN checking that a case has already been uploaded to Loqusdb - is_duplicate: bool = mip_dna_observations_api.is_duplicate( - case=case, - loqusdb_api=mip_dna_observations_api.loqusdb_api, - profile_vcf_path=observations_input_files.profile_vcf_path, - profile_threshold=MipDNALoadParameters.PROFILE_THRESHOLD.value, - ) - - # THEN an upload of a duplicate case should be detected - assert is_duplicate is True - - -def test_is_duplicate_loqusdb_id( - case_id: str, - loqusdb_id: str, - mip_dna_observations_api: MipDNAObservationsAPI, - observations_input_files: MipDNAObservationsInputFiles, - mocker, -): - """Test duplicate extraction for a case that already exists in Loqusdb.""" - - # GIVEN a Loqusdb instance with a duplicated case and whose samples already have a Loqusdb ID - case: Case = mip_dna_observations_api.store.get_case_by_internal_id(internal_id=case_id) - case.links[0].sample.loqusdb_id = loqusdb_id - mocker.patch.object(mip_dna_observations_api.loqusdb_api, "get_case", return_value=None) - mocker.patch.object(mip_dna_observations_api.loqusdb_api, "get_duplicate", return_value=False) - - # WHEN checking that the sample observations have already been uploaded - is_duplicate: bool = mip_dna_observations_api.is_duplicate( - case=case, - loqusdb_api=mip_dna_observations_api.loqusdb_api, - profile_vcf_path=observations_input_files.profile_vcf_path, - profile_threshold=MipDNALoadParameters.PROFILE_THRESHOLD.value, - ) - - # THEN a duplicated upload should be identified - assert is_duplicate is True - - -def test_check_customer_loqusdb_permissions( - customer_rare_diseases: Customer, - customer_balsamic: Customer, - mip_dna_observations_api: MipDNAObservationsAPI, - caplog: LogCaptureFixture, -): - """Test customers Loqusdb permissions.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a MIP observations API, a Rare Disease customer and a Cancer customer - - # WHEN verifying the permissions for Loqusdb upload - mip_dna_observations_api.check_customer_loqusdb_permissions(customer_rare_diseases) - - # THEN it should be only possible to upload data from a RD customer - assert f"Valid customer {customer_rare_diseases.internal_id} for Loqusdb uploads" in caplog.text - with pytest.raises(LoqusdbUploadCaseError): - mip_dna_observations_api.check_customer_loqusdb_permissions(customer_balsamic) - - -def test_mip_dna_get_loqusdb_instance(mip_dna_observations_api: MipDNAObservationsAPI): - """Test Loqusdb instance retrieval given a sequencing method.""" - - # GIVEN a rare disease observations API with a WES as sequencing method - mip_dna_observations_api.sequencing_method = SequencingMethod.WES - - # WHEN getting the Loqusdb instance - loqusdb_instance = mip_dna_observations_api.get_loqusdb_instance() - - # THEN the correct loqusdb instance should be returned - assert loqusdb_instance == LoqusdbInstance.WES - - -def test_mip_dna_get_loqusdb_instance_not_supported( - mip_dna_observations_api: MipDNAObservationsAPI, caplog: LogCaptureFixture -): - """Test Loqusdb instance retrieval given a not supported sequencing method.""" - - # GIVEN a rare disease observations API with a WTS sequencing method - mip_dna_observations_api.sequencing_method = SequencingMethod.WTS - - # WHEN getting the Loqusdb instance - with pytest.raises(LoqusdbUploadCaseError): - # THEN the upload should be canceled - mip_dna_observations_api.get_loqusdb_instance() - - assert ( - f"Sequencing method {SequencingMethod.WTS} is not supported by Loqusdb. Cancelling upload." - in caplog.text - ) - - -def test_mip_dna_load_observations( - case_id: str, - mip_dna_observations_api: MipDNAObservationsAPI, - observations_input_files: MipDNAObservationsInputFiles, - nr_of_loaded_variants: int, - caplog: LogCaptureFixture, - mocker, -): - """Test loading of case observations for rare disease.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a mock MIP DNA observations API and a list of observations input files - case: Case = mip_dna_observations_api.store.get_case_by_internal_id(internal_id=case_id) - mocker.patch.object(mip_dna_observations_api, "is_duplicate", return_value=False) - - # WHEN loading the case to Loqusdb - mip_dna_observations_api.load_observations(case, observations_input_files) - - # THEN the observations should be loaded without any errors - assert f"Uploaded {nr_of_loaded_variants} variants to Loqusdb" in caplog.text - - -def test_mip_dna_load_observations_duplicate( - case_id: str, - mip_dna_observations_api: MipDNAObservationsAPI, - observations_input_files: MipDNAObservationsInputFiles, - caplog: LogCaptureFixture, - mocker, -): - """Test upload case duplicate to Loqusdb.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a mocked observations API and a case object that has already been uploaded to Loqusdb - case: Case = mip_dna_observations_api.store.get_case_by_internal_id(internal_id=case_id) - mocker.patch.object(mip_dna_observations_api, "is_duplicate", return_value=True) - - # WHEN uploading the case observations to Loqusdb - with pytest.raises(LoqusdbDuplicateRecordError): - # THEN a duplicate record error should be raised - mip_dna_observations_api.load_observations(case, observations_input_files) - - assert f"Case {case.internal_id} has already been uploaded to Loqusdb" in caplog.text - - -def test_mip_dna_load_observations_tumor_case( - case_id: str, - mip_dna_observations_api: MipDNAObservationsAPI, - observations_input_files: MipDNAObservationsInputFiles, - caplog: LogCaptureFixture, - mocker, -): - """Test loading of a tumor case to Loqusdb.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a MIP DNA observations API and a case object with a tumour sample - case: Case = mip_dna_observations_api.store.get_case_by_internal_id(internal_id=case_id) - mocker.patch.object(mip_dna_observations_api, "is_duplicate", return_value=False) - case.links[0].sample.is_tumour = True - - # WHEN getting the Loqusdb API - with pytest.raises(LoqusdbUploadCaseError): - # THEN an upload error should be raised and the execution aborted - mip_dna_observations_api.load_observations(case, observations_input_files) - - assert f"Case {case.internal_id} has tumour samples. Cancelling upload." in caplog.text - - -def test_mip_dna_delete_case( - case_id: str, - mip_dna_observations_api: MipDNAObservationsAPI, - caplog: LogCaptureFixture, -): - """Test delete case from Loqusdb.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a Loqusdb instance filled with a case - case: Case = mip_dna_observations_api.store.get_case_by_internal_id(internal_id=case_id) - - # WHEN deleting a case - mip_dna_observations_api.delete_case(case) - - # THEN the case should be deleted from Loqusdb - assert f"Removed observations for case {case.internal_id} from Loqusdb" in caplog.text - - -def test_mip_dna_delete_case_not_found( - helpers: StoreHelpers, - loqusdb_api: LoqusdbAPI, - mip_dna_observations_api: MipDNAObservationsAPI, - caplog: LogCaptureFixture, -): - """Test delete case from Loqusdb that has not been uploaded.""" - - # GIVEN an observations instance and a case that has not been uploaded to Loqusdb - loqusdb_api.process.stdout = None - mip_dna_observations_api.loqusdb_api = loqusdb_api - case: Case = helpers.add_case(mip_dna_observations_api.store) - - # WHEN deleting a rare disease case that does not exist in Loqusdb - with pytest.raises(CaseNotFoundError): - # THEN a CaseNotFoundError should be raised - mip_dna_observations_api.delete_case(case) - - assert ( - f"Case {case.internal_id} could not be found in Loqusdb. Skipping case deletion." - in caplog.text - ) - - -def test_balsamic_load_observations( - case_id: str, - balsamic_observations_api: BalsamicObservationsAPI, - balsamic_observations_input_files: BalsamicObservationsInputFiles, - nr_of_loaded_variants: int, - caplog: LogCaptureFixture, - mocker, -): - """Test loading of cancer case observations.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a mock BALSAMIC observations API and a list of observations input files - case: Case = balsamic_observations_api.store.get_case_by_internal_id(internal_id=case_id) - mocker.patch.object(balsamic_observations_api, "is_duplicate", return_value=False) - - # WHEN loading the case to Loqusdb - balsamic_observations_api.load_observations(case, balsamic_observations_input_files) - - # THEN the observations should be loaded successfully - assert f"Uploaded {nr_of_loaded_variants} variants to Loqusdb" in caplog.text - - -def test_balsamic_load_observations_duplicate( - case_id: str, - mip_dna_observations_api: MipDNAObservationsAPI, - observations_input_files: MipDNAObservationsInputFiles, - caplog: LogCaptureFixture, - mocker, -): - """Test upload cancer duplicate case observations to Loqusdb.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a balsamic observations API and a case object that has already been uploaded to Loqusdb - case: Case = mip_dna_observations_api.store.get_case_by_internal_id(internal_id=case_id) - mocker.patch.object(mip_dna_observations_api, "is_duplicate", return_value=True) - - # WHEN uploading the case observations to Loqusdb - with pytest.raises(LoqusdbDuplicateRecordError): - # THEN a duplicate record error should be raised - mip_dna_observations_api.load_observations(case, observations_input_files) - - assert f"Case {case.internal_id} has already been uploaded to Loqusdb" in caplog.text - - -def test_balsamic_load_cancer_observations( - case_id: str, - balsamic_observations_api: BalsamicObservationsAPI, - balsamic_observations_input_files: BalsamicObservationsInputFiles, - nr_of_loaded_variants: int, - caplog: LogCaptureFixture, -): - """Test loading of case observations for cancer.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a mock BALSAMIC observations API and a list of observations input files - case: Case = balsamic_observations_api.store.get_case_by_internal_id(internal_id=case_id) - - # WHEN loading the case to a somatic Loqusdb instance - balsamic_observations_api.load_cancer_observations( - case, balsamic_observations_input_files, balsamic_observations_api.loqusdb_somatic_api - ) - - # THEN the observations should be loaded successfully - assert f"Uploaded {nr_of_loaded_variants} variants to Loqusdb" in caplog.text - print(caplog.text) - - -def test_balsamic_delete_case( - case_id: str, - balsamic_observations_api: BalsamicObservationsAPI, - caplog: LogCaptureFixture, -): - """Test delete balsamic case observations from Loqusdb.""" - caplog.set_level(logging.DEBUG) - - # GIVEN a Loqusdb instance and a case that has been uploaded to both somatic and tumor instances - case: Case = balsamic_observations_api.store.get_case_by_internal_id(internal_id=case_id) - - # WHEN deleting the case - balsamic_observations_api.delete_case(case) - - # THEN the case should be deleted from Loqusdb - assert f"Removed observations for case {case.internal_id} from Loqusdb" in caplog.text - - -def test_balsamic_delete_case_not_found( - helpers: StoreHelpers, - loqusdb_api: LoqusdbAPI, - balsamic_observations_api: BalsamicObservationsAPI, - caplog: LogCaptureFixture, -): - """Test delete balsamic case observations from Loqusdb that have not been uploaded.""" - - # GIVEN empty Loqusdb instances - loqusdb_api.process.stdout = None - balsamic_observations_api.loqusdb_somatic_api = loqusdb_api - balsamic_observations_api.loqusdb_tumor_api = loqusdb_api - case: Case = helpers.add_case(balsamic_observations_api.store) - - # WHEN deleting a cancer case that does not exist in Loqusdb - with pytest.raises(CaseNotFoundError): - # THEN a CaseNotFoundError should be raised - balsamic_observations_api.delete_case(case) - - assert ( - f"Case {case.internal_id} could not be found in Loqusdb. Skipping case deletion." - in caplog.text - ) diff --git a/tests/meta/observations/test_mip_dna_observations_api.py b/tests/meta/observations/test_mip_dna_observations_api.py new file mode 100644 index 0000000000..db7e9e8f85 --- /dev/null +++ b/tests/meta/observations/test_mip_dna_observations_api.py @@ -0,0 +1,160 @@ +"""Test MIP-DNA observations API.""" + +import logging + +import pytest +from _pytest.logging import LogCaptureFixture +from pytest_mock import MockFixture + +from cg.constants.sequencing import SequencingMethod +from cg.exc import LoqusdbDuplicateRecordError +from cg.meta.observations.mip_dna_observations_api import MipDNAObservationsAPI +from cg.meta.workflow.mip_dna import MipDNAAnalysisAPI +from cg.models.observations.input_files import MipDNAObservationsInputFiles +from cg.store.models import Case + + +def test_is_sample_type_eligible_for_observations_upload( + case_id: str, mip_dna_observations_api: MipDNAObservationsAPI +): + """Test if the sample type is eligible for observation uploads.""" + + # GIVEN a case without tumor samples and a MIP-DNA observations API + case: Case = mip_dna_observations_api.store.get_case_by_internal_id(case_id) + + # WHEN checking sample type eligibility for a case + is_sample_type_eligible_for_observations_upload: bool = ( + mip_dna_observations_api.is_sample_type_eligible_for_observations_upload(case) + ) + + # THEN the analysis type should be eligible for observation uploads + assert is_sample_type_eligible_for_observations_upload + + +def test_is_sample_type_not_eligible_for_observations_upload( + case_id: str, mip_dna_observations_api: MipDNAObservationsAPI +): + """Test if the sample type is not eligible for observation uploads.""" + + # GIVEN a case with tumor samples and a MIP-DNA observations API + case: Case = mip_dna_observations_api.store.get_case_by_internal_id(case_id) + case.samples[0].is_tumour = True + + # WHEN checking sample type eligibility for a case + is_sample_type_eligible_for_observations_upload: bool = ( + mip_dna_observations_api.is_sample_type_eligible_for_observations_upload(case) + ) + + # THEN the analysis type should not be eligible for observation uploads + assert not is_sample_type_eligible_for_observations_upload + + +def test_is_case_eligible_for_observations_upload( + case_id: str, mip_dna_observations_api: MipDNAObservationsAPI, mocker: MockFixture +): + """Test whether a case is eligible for MIP-DNA observation uploads.""" + + # GIVEN a case and a MIP-DNA observations API + case: Case = mip_dna_observations_api.analysis_api.status_db.get_case_by_internal_id(case_id) + + # GIVEN a MIP-DNA scenario for Loqusdb uploads + mocker.patch.object( + MipDNAAnalysisAPI, "get_data_analysis_type", return_value=SequencingMethod.WGS + ) + + # WHEN checking the upload eligibility for a case + is_case_eligible_for_observations_upload: bool = ( + mip_dna_observations_api.is_case_eligible_for_observations_upload(case) + ) + + # THEN the case should be eligible for observation uploads + assert is_case_eligible_for_observations_upload + + +def test_is_case_not_eligible_for_observations_upload( + case_id: str, mip_dna_observations_api: MipDNAObservationsAPI, mocker: MockFixture +): + """Test whether a case is not eligible for MIP-DNA observation uploads.""" + + # GIVEN a case and a MIP-DNA observations API + case: Case = mip_dna_observations_api.analysis_api.status_db.get_case_by_internal_id(case_id) + + # GIVEN a MIP-DNA scenario for Loqusdb uploads with an invalid sequencing method + mocker.patch.object( + MipDNAAnalysisAPI, "get_data_analysis_type", return_value=SequencingMethod.WTS + ) + + # WHEN checking the upload eligibility for a case + is_case_eligible_for_observations_upload: bool = ( + mip_dna_observations_api.is_case_eligible_for_observations_upload(case) + ) + + # THEN the case should not be eligible for observation uploads + assert not is_case_eligible_for_observations_upload + + +def test_load_observations( + case_id: str, + loqusdb_id: str, + number_of_loaded_variants: int, + mip_dna_observations_api: MipDNAObservationsAPI, + mip_dna_observations_input_files: MipDNAObservationsInputFiles, + caplog: LogCaptureFixture, + mocker: MockFixture, +): + """Test loading of case observations for MIP-DNA.""" + caplog.set_level(logging.DEBUG) + + # GIVEN a MIP-DNA observations API and a list of observations input files + + # GIVEN a MIP-DNA case and a mocked scenario for uploads + case: Case = mip_dna_observations_api.store.get_case_by_internal_id(case_id) + mocker.patch.object( + MipDNAAnalysisAPI, "get_data_analysis_type", return_value=SequencingMethod.WGS + ) + mocker.patch.object( + MipDNAObservationsAPI, + "get_observations_input_files", + return_value=mip_dna_observations_input_files, + ) + mocker.patch.object(MipDNAObservationsAPI, "is_duplicate", return_value=False) + + # WHEN loading the case to Loqusdb + mip_dna_observations_api.load_observations(case) + + # THEN the observations should be loaded without any errors + assert f"Uploaded {number_of_loaded_variants} variants to Loqusdb" in caplog.text + + +def test_load_duplicated_observations( + case_id: str, + loqusdb_id: str, + number_of_loaded_variants: int, + mip_dna_observations_api: MipDNAObservationsAPI, + mip_dna_observations_input_files: MipDNAObservationsInputFiles, + caplog: LogCaptureFixture, + mocker: MockFixture, +): + """Test loading of duplicated observations for MIP-DNA.""" + caplog.set_level(logging.DEBUG) + + # GIVEN a MIP-DNA observations API and a list of observations input files + + # GIVEN a MIP-DNA case and a mocked scenario for uploads with a case already uploaded + case: Case = mip_dna_observations_api.store.get_case_by_internal_id(case_id) + mocker.patch.object( + MipDNAAnalysisAPI, "get_data_analysis_type", return_value=SequencingMethod.WGS + ) + mocker.patch.object( + MipDNAObservationsAPI, + "get_observations_input_files", + return_value=mip_dna_observations_input_files, + ) + mocker.patch.object(MipDNAObservationsAPI, "is_duplicate", return_value=True) + + # WHEN loading the case to Loqusdb + with pytest.raises(LoqusdbDuplicateRecordError): + mip_dna_observations_api.load_observations(case) + + # THEN the observations upload should be aborted + assert f"Case {case_id} has already been uploaded to Loqusdb" in caplog.text diff --git a/tests/meta/observations/test_observations_api.py b/tests/meta/observations/test_observations_api.py new file mode 100644 index 0000000000..0c5a41fd13 --- /dev/null +++ b/tests/meta/observations/test_observations_api.py @@ -0,0 +1,497 @@ +"""Test observations API methods.""" + +import logging +from pathlib import Path + +import pytest +from _pytest.fixtures import FixtureRequest +from _pytest.logging import LogCaptureFixture +from pytest_mock import MockFixture + +from cg.apps.lims import LimsAPI +from cg.apps.loqus import LoqusdbAPI +from cg.constants.constants import CancerAnalysisType, CustomerId, Workflow +from cg.constants.observations import LoqusdbInstance, MipDNALoadParameters +from cg.constants.sample_sources import SourceType +from cg.constants.sequencing import SequencingMethod +from cg.exc import CaseNotFoundError, LoqusdbUploadCaseError +from cg.meta.observations.observations_api import ObservationsAPI +from cg.meta.workflow.analysis import AnalysisAPI +from cg.meta.workflow.balsamic import BalsamicAnalysisAPI +from cg.meta.workflow.mip_dna import MipDNAAnalysisAPI +from cg.models.cg_config import CGConfig +from cg.models.observations.input_files import ObservationsInputFiles +from cg.store.models import Case, Customer + + +@pytest.mark.parametrize( + "workflow, analysis_api, sequencing_method, is_eligible, expected_message", + [ + ( + Workflow.BALSAMIC, + BalsamicAnalysisAPI, + CancerAnalysisType.TUMOR_WGS, + True, + "Uploaded {number_of_loaded_variants} variants to Loqusdb", + ), + ( + Workflow.MIP_DNA, + MipDNAAnalysisAPI, + SequencingMethod.WGS, + True, + "Uploaded {number_of_loaded_variants} variants to Loqusdb", + ), + ( + Workflow.BALSAMIC, + BalsamicAnalysisAPI, + CancerAnalysisType.TUMOR_WGS, + False, + "Case {case_id} is not eligible for observations upload", + ), + ( + Workflow.MIP_DNA, + MipDNAAnalysisAPI, + SequencingMethod.WGS, + False, + "Case {case_id} is not eligible for observations upload", + ), + ], +) +def test_observations_upload( + case_id: str, + loqusdb_id: str, + number_of_loaded_variants: int, + workflow: Workflow, + analysis_api: AnalysisAPI, + sequencing_method: str, + is_eligible: bool, + expected_message: str, + request: FixtureRequest, + mocker: MockFixture, + caplog: LogCaptureFixture, +): + """Test upload of observations.""" + caplog.set_level(logging.DEBUG) + + # GIVEN an observations API and a list of observation input files + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + observations_input_files: ObservationsInputFiles = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_input_files" + ) + + # GIVEN a mock scenario for a successful upload + mocker.patch.object(analysis_api, "get_data_analysis_type", return_value=sequencing_method) + mocker.patch.object( + ObservationsAPI, "get_observations_input_files", return_value=observations_input_files + ) + mocker.patch.object(ObservationsAPI, "is_duplicate", return_value=False) + + # GIVEN a case not eligible for Loqusdb uploads + if not is_eligible: + mocker.patch.object(LimsAPI, "get_source", return_value=SourceType.TISSUE_FFPE) + + # WHEN uploading the case observations to Loqusdb + if is_eligible: + observations_api.upload(case_id) + else: + with pytest.raises(LoqusdbUploadCaseError): + observations_api.upload(case_id) + + # THEN the expected log message should be present + assert ( + expected_message.format( + case_id=case_id, number_of_loaded_variants=number_of_loaded_variants + ) + in caplog.text + ) + + +@pytest.mark.parametrize( + "workflow, loqusdb_instance", + [(Workflow.BALSAMIC, LoqusdbInstance.TUMOR), (Workflow.MIP_DNA, LoqusdbInstance.WES)], +) +def test_get_loqusdb_api( + cg_context: CGConfig, + workflow: Workflow, + loqusdb_instance: LoqusdbInstance, + request: FixtureRequest, +): + """Test Loqusdb API retrieval given a Loqusdb instance.""" + + # GIVEN a WES Loqusdb instance and an observations API + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + + # GIVEN the expected Loqusdb config dictionary + loqusdb_instance_key: str = loqusdb_instance.value.replace("-", "_") + loqusdb_config: dict[str, Path] = cg_context.dict()[loqusdb_instance_key] + + # WHEN calling the Loqusdb API get method + loqusdb_api: LoqusdbAPI = observations_api.get_loqusdb_api(loqusdb_instance) + + # THEN a WES Loqusdb API should be returned + assert isinstance(loqusdb_api, LoqusdbAPI) + assert loqusdb_api.binary_path == loqusdb_config["binary_path"] + assert loqusdb_api.config_path == loqusdb_config["config_path"] + + +@pytest.mark.parametrize( + "workflow, loqusdb_instance", + [(Workflow.BALSAMIC, LoqusdbInstance.TUMOR), (Workflow.MIP_DNA, LoqusdbInstance.WES)], +) +def test_is_not_duplicate( + case_id: str, + workflow: Workflow, + loqusdb_instance: LoqusdbInstance, + request: FixtureRequest, + mocker: MockFixture, +): + """Test duplicate extraction for a case that is not in Loqusdb.""" + + # GIVEN an observations API and a list of files to upload + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + observations_input_files: ObservationsInputFiles = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_input_files" + ) + + # GIVEN a Loqusdb instance with no case duplicates + case: Case = observations_api.store.get_case_by_internal_id(case_id) + loqusdb_api: LoqusdbAPI = observations_api.get_loqusdb_api(loqusdb_instance) + mocker.patch.object(LoqusdbAPI, "get_case", return_value=None) + mocker.patch.object(LoqusdbAPI, "get_duplicate", return_value=False) + + # WHEN checking that case has not been uploaded to Loqusdb + is_duplicate: bool = observations_api.is_duplicate( + case=case, + loqusdb_api=loqusdb_api, + profile_vcf_path=observations_input_files.snv_vcf_path, + profile_threshold=MipDNALoadParameters.PROFILE_THRESHOLD.value, + ) + + # THEN there should be no duplicates in Loqusdb + assert is_duplicate is False + + +@pytest.mark.parametrize( + "workflow, loqusdb_instance", + [(Workflow.BALSAMIC, LoqusdbInstance.TUMOR), (Workflow.MIP_DNA, LoqusdbInstance.WES)], +) +def test_is_duplicate( + case_id: str, + workflow: Workflow, + loqusdb_instance: LoqusdbInstance, + request: FixtureRequest, + mocker: MockFixture, +): + """Test duplicate extraction for a case that already exists in Loqusdb.""" + + # GIVEN an observations API and a list of files to upload + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + observations_input_files: ObservationsInputFiles = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_input_files" + ) + + # GIVEN a Loqusdb instance with a duplicated case + case: Case = observations_api.store.get_case_by_internal_id(case_id) + loqusdb_api: LoqusdbAPI = observations_api.get_loqusdb_api(loqusdb_instance) + mocker.patch.object(LoqusdbAPI, "get_case", return_value=None) + mocker.patch.object(LoqusdbAPI, "get_duplicate", return_value={"case_id": case_id}) + + # WHEN checking that case has not been uploaded to Loqusdb + is_duplicate: bool = observations_api.is_duplicate( + case=case, + loqusdb_api=loqusdb_api, + profile_vcf_path=observations_input_files.snv_vcf_path, + profile_threshold=MipDNALoadParameters.PROFILE_THRESHOLD.value, + ) + + # THEN an upload of a duplicate case should be detected + assert is_duplicate is True + + +@pytest.mark.parametrize( + "workflow, loqusdb_instance", + [(Workflow.BALSAMIC, LoqusdbInstance.TUMOR), (Workflow.MIP_DNA, LoqusdbInstance.WES)], +) +def test_is_duplicate_loqusdb_id( + case_id: str, + loqusdb_id: str, + workflow: Workflow, + loqusdb_instance: LoqusdbInstance, + request: FixtureRequest, + mocker: MockFixture, +): + """Test duplicate extraction for a case that already exists in Loqusdb.""" + + # GIVEN an observations API and a list of files to upload + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + observations_input_files: ObservationsInputFiles = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_input_files" + ) + + # GIVEN a Loqusdb instance with a duplicated case and whose samples already have a Loqusdb ID + case: Case = observations_api.store.get_case_by_internal_id(case_id) + loqusdb_api: LoqusdbAPI = observations_api.get_loqusdb_api(loqusdb_instance) + case.links[0].sample.loqusdb_id = loqusdb_id + mocker.patch.object(LoqusdbAPI, "get_case", return_value=None) + mocker.patch.object(LoqusdbAPI, "get_duplicate", return_value=False) + + # WHEN checking that the sample observations have already been uploaded + is_duplicate: bool = observations_api.is_duplicate( + case=case, + loqusdb_api=loqusdb_api, + profile_vcf_path=observations_input_files.snv_vcf_path, + profile_threshold=MipDNALoadParameters.PROFILE_THRESHOLD.value, + ) + + # THEN a duplicated upload should be identified + assert is_duplicate is True + + +@pytest.mark.parametrize("workflow", [Workflow.BALSAMIC, Workflow.MIP_DNA]) +def test_is_customer_eligible_for_observations_upload( + workflow: Workflow, + request: FixtureRequest, +): + """Test if customer is eligible for observations upload.""" + + # GIVEN a MIP-DNA customer and observations API + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + customer: Customer = request.getfixturevalue(f"{workflow.replace('-', '_')}_customer") + customer_id: str = customer.internal_id + + # WHEN verifying if the customer is eligible for Balsamic observations upload + is_customer_eligible_for_observations_upload: bool = ( + observations_api.is_customer_eligible_for_observations_upload(customer_id) + ) + + # THEN the customer's data should be eligible for uploads + assert is_customer_eligible_for_observations_upload + + +@pytest.mark.parametrize("workflow", [Workflow.BALSAMIC, Workflow.MIP_DNA]) +def test_is_customer_not_eligible_for_observations_upload( + workflow: Workflow, request: FixtureRequest, caplog: LogCaptureFixture +): + """Test if customer is not eligible for observations upload.""" + + # GIVEN a CG internal customer ID and observations API + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + customer_id: str = CustomerId.CG_INTERNAL_CUSTOMER + + # WHEN verifying if the customer is eligible for Balsamic observations upload + is_customer_eligible_for_observations_upload: bool = ( + observations_api.is_customer_eligible_for_observations_upload(customer_id) + ) + + # THEN the customer's data should not be eligible for uploads + assert not is_customer_eligible_for_observations_upload + assert f"Customer {customer_id} is not whitelisted for Loqusdb uploads" in caplog.text + + +@pytest.mark.parametrize( + "workflow, analysis_api, sequencing_method", + [ + (Workflow.BALSAMIC, BalsamicAnalysisAPI, CancerAnalysisType.TUMOR_WGS), + (Workflow.MIP_DNA, MipDNAAnalysisAPI, SequencingMethod.WGS), + ], +) +def test_is_sequencing_method_eligible_for_observations_upload( + case_id: str, + workflow: Workflow, + analysis_api: AnalysisAPI, + sequencing_method: str, + request: FixtureRequest, + mocker: MockFixture, +): + """Test if the sequencing method is eligible for observations uploads.""" + + # GIVEN a case ID and an observations API + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + + # GIVEN a supported data analysis type + mocker.patch.object(analysis_api, "get_data_analysis_type", return_value=sequencing_method) + + # WHEN verifying that the sequencing method is eligible for observations uploads + is_sequencing_method_eligible_for_observations_upload: bool = ( + observations_api.is_sequencing_method_eligible_for_observations_upload(case_id) + ) + + # THEN the sequencing method should be eligible for observations uploads + assert is_sequencing_method_eligible_for_observations_upload + + +@pytest.mark.parametrize( + "workflow, analysis_api, sequencing_method", + [ + (Workflow.BALSAMIC, BalsamicAnalysisAPI, CancerAnalysisType.TUMOR_PANEL), + (Workflow.MIP_DNA, MipDNAAnalysisAPI, SequencingMethod.WTS), + ], +) +def test_is_sequencing_method_not_eligible_for_observations_upload( + case_id: str, + workflow: Workflow, + analysis_api: AnalysisAPI, + sequencing_method: str, + request: FixtureRequest, + mocker: MockFixture, + caplog: LogCaptureFixture, +): + """Test if the sequencing method is eligible for observations uploads.""" + + # GIVEN a case ID and an observations API + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + # GIVEN a non-supported data analysis type + mocker.patch.object(analysis_api, "get_data_analysis_type", return_value=sequencing_method) + + # WHEN verifying that the sequencing method is eligible for observations uploads + is_sequencing_method_eligible_for_observations_upload: bool = ( + observations_api.is_sequencing_method_eligible_for_observations_upload(case_id) + ) + + # THEN the sequencing method should not be eligible for observations uploads + assert not is_sequencing_method_eligible_for_observations_upload + assert ( + f"Sequencing method {sequencing_method} is not supported by Loqusdb uploads" in caplog.text + ) + + +@pytest.mark.parametrize("workflow", [Workflow.BALSAMIC, Workflow.MIP_DNA]) +def test_is_sample_source_eligible_for_observations_upload( + case_id: str, workflow: Workflow, request: FixtureRequest, mocker: MockFixture +): + """Test if the sample source is eligible for observations uploads.""" + + # GIVEN a case ID and an observations API + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + + # GIVEN a supported sample source + + # WHEN verifying that the sample source is eligible for observations uploads + is_sample_source_eligible_for_observations_upload: bool = ( + observations_api.is_sample_source_eligible_for_observations_upload(case_id) + ) + + # THEN the source type should be eligible for observations uploads + assert is_sample_source_eligible_for_observations_upload + + +@pytest.mark.parametrize("workflow", [Workflow.BALSAMIC, Workflow.MIP_DNA]) +def test_is_sample_source_not_eligible_for_observations_upload( + case_id: str, + workflow: Workflow, + request: FixtureRequest, + mocker: MockFixture, + caplog: LogCaptureFixture, +): + """Test if the sample source is not eligible for observations uploads.""" + + # GIVEN a case ID and an observations API + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + + # GIVEN a not supported sample source + source_type = SourceType.TISSUE_FFPE + mocker.patch.object(LimsAPI, "get_source", return_value=source_type) + + # WHEN verifying that the sample source is eligible for observations uploads + is_sample_source_eligible_for_observations_upload: bool = ( + observations_api.is_sample_source_eligible_for_observations_upload(case_id) + ) + + # THEN the source type should not be eligible for observations uploads + assert not is_sample_source_eligible_for_observations_upload + assert f"Source type {source_type} is not supported for Loqusdb uploads" in caplog.text + + +@pytest.mark.parametrize( + "workflow, analysis_api, sequencing_method", + [ + (Workflow.BALSAMIC, BalsamicAnalysisAPI, CancerAnalysisType.TUMOR_WGS), + (Workflow.MIP_DNA, MipDNAAnalysisAPI, SequencingMethod.WGS), + ], +) +def test_delete_case( + case_id: str, + loqusdb_id: str, + workflow: Workflow, + analysis_api: AnalysisAPI, + sequencing_method: str, + request: FixtureRequest, + mocker: MockFixture, + caplog: LogCaptureFixture, +): + """Test delete case from Loqusdb.""" + caplog.set_level(logging.DEBUG) + + # GIVEN a case ID and an observations API + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + + # GIVEN a case uploaded to Loqusdb + mocker.patch.object(analysis_api, "get_data_analysis_type", return_value=sequencing_method) + mocker.patch.object(LoqusdbAPI, "delete_case", return_value=None) + + # WHEN deleting a case + observations_api.delete_case(case_id) + + # THEN the case should be deleted from Loqusdb + assert f"Removed observations for case {case_id}" in caplog.text + + +@pytest.mark.parametrize( + "workflow, analysis_api, sequencing_method", + [ + (Workflow.BALSAMIC, BalsamicAnalysisAPI, CancerAnalysisType.TUMOR_WGS), + (Workflow.MIP_DNA, MipDNAAnalysisAPI, SequencingMethod.WGS), + ], +) +def test_delete_case_not_found( + case_id: str, + loqusdb_id: str, + workflow: Workflow, + analysis_api: AnalysisAPI, + sequencing_method: str, + request: FixtureRequest, + mocker: MockFixture, + caplog: LogCaptureFixture, +): + """Test delete case from Loqusdb that has not been uploaded.""" + caplog.set_level(logging.DEBUG) + + # GIVEN a case ID and an observations API + observations_api: ObservationsAPI = request.getfixturevalue( + f"{workflow.replace('-', '_')}_observations_api" + ) + + # GIVEN a case that has not been uploaded to Loqusdb + mocker.patch.object(LoqusdbAPI, "get_case", return_value=None) + mocker.patch.object(analysis_api, "get_data_analysis_type", return_value=sequencing_method) + + # WHEN deleting a case + with pytest.raises(CaseNotFoundError): + observations_api.delete_case(case_id) + + # THEN a CaseNotFoundError should be raised + assert f"Case {case_id} could not be found in Loqusdb. Skipping case deletion." in caplog.text diff --git a/tests/mocks/limsmock.py b/tests/mocks/limsmock.py index 1768c37383..97eb1f7eaf 100644 --- a/tests/mocks/limsmock.py +++ b/tests/mocks/limsmock.py @@ -55,6 +55,7 @@ def __init__(self, config: dict = None, samples: list[dict] = None): ) self._sequencing_method = "CG002 - Cluster Generation (HiSeq X)" self._delivery_method = "CG002 - Delivery" + self._source = "cell-free DNA" def set_prep_method(self, method: str = "1337:00 Test prep method"): """Mock function""" @@ -64,8 +65,8 @@ def set_sequencing_method(self, method: str = "1338:00 Test sequencing method"): """Mock function""" self._prep_method = method - def sample(self, sample_id: str) -> dict | None: - return next((sample for sample in self._samples if sample["id"] == sample_id), None) + def sample(self, lims_id: str) -> dict: + return next((sample for sample in self._samples if sample["id"] == lims_id), {}) def add_sample(self, internal_id: str): self.sample_vars[internal_id] = {} @@ -89,6 +90,9 @@ def get_sequencing_method(self, lims_id: str) -> str: def get_delivery_method(self, lims_id: str) -> str: return self._delivery_method + def get_source(self, lims_id: str) -> str | None: + return self._source + def get_sample_comment(self, sample_id: str) -> str: lims_sample: dict[str, Any] = self.sample(sample_id) comment = None diff --git a/tests/models/observations/test_observations_input_files.py b/tests/models/observations/test_observations_input_files.py index e0f2c518a5..ceacdec736 100644 --- a/tests/models/observations/test_observations_input_files.py +++ b/tests/models/observations/test_observations_input_files.py @@ -11,35 +11,35 @@ ) -def test_instantiate_input_files(observations_input_files_raw: dict): +def test_instantiate_input_files(mip_dna_observations_input_files_raw: dict[str, Path]): """Tests input files against a pydantic MipDNAObservationsInputFiles.""" # GIVEN a dictionary with the basic input files # WHEN instantiating an observations input files object - input_files = MipDNAObservationsInputFiles(**observations_input_files_raw) + input_files = MipDNAObservationsInputFiles(**mip_dna_observations_input_files_raw) # THEN assert that it was successfully created assert isinstance(input_files, MipDNAObservationsInputFiles) def test_instantiate_input_files_missing_field( - observations_input_files_raw: dict, file_does_not_exist: Path + mip_dna_observations_input_files_raw: dict[str, Path], file_does_not_exist: Path ): """Tests input files against a pydantic MipDNAObservationsInputFiles with not existent field.""" # GIVEN a dictionary with the basic input files and a file path that does not exist - observations_input_files_raw["snv_vcf_path"] = file_does_not_exist + mip_dna_observations_input_files_raw["snv_vcf_path"] = file_does_not_exist # WHEN checking the observation file # THEN the file is not successfully validated and an error is returned with pytest.raises(ValidationError): # WHEN instantiating a ObservationsInputFiles object - MipDNAObservationsInputFiles(**observations_input_files_raw) + MipDNAObservationsInputFiles(**mip_dna_observations_input_files_raw) -def test_instantiate_balsamic_input_files(balsamic_observations_input_files_raw: dict): +def test_instantiate_balsamic_input_files(balsamic_observations_input_files_raw: dict[str, Path]): """Tests input files against a pydantic BalsamicObservationsInputFiles.""" # GIVEN balsamic input files @@ -52,7 +52,7 @@ def test_instantiate_balsamic_input_files(balsamic_observations_input_files_raw: def test_instantiate_balsamic_input_files_missing_field( - balsamic_observations_input_files_raw: dict, file_does_not_exist: Path + balsamic_observations_input_files_raw: dict[str, Path], file_does_not_exist: Path ): """Tests input files against a pydantic BalsamicObservationsInputFiles with not existent field.""" diff --git a/tests/store/conftest.py b/tests/store/conftest.py index e45b7b1c2d..752af58ddd 100644 --- a/tests/store/conftest.py +++ b/tests/store/conftest.py @@ -31,7 +31,7 @@ class StoreConstants(enum.Enum): CUSTOMER_ID_SAMPLE_WITH_ATTRIBUTES: str = "1" SUBJECT_ID_SAMPLE_WITH_ATTRIBUTES: str = "test_subject_id" ORGANISM_ID_SAMPLE_WITH_ATTRIBUTES: int = 1 - LOCUSDB_ID_SAMPLE_WITH_ATTRIBUTES: str = "locusdb_id" + LOQUSDB_ID_SAMPLE_WITH_ATTRIBUTES: str = "loqusdb_id" READS_SAMPLE_WITH_ATTRIBUTES: int = 100 DOWN_SAMPLED_TO_SAMPLE_WITH_ATTRIBUTES: int = 1 AGE_AT_SAMPLING_SAMPLE_WITH_ATTRIBUTES: float = 45 @@ -227,7 +227,7 @@ def store_with_a_sample_that_has_many_attributes_and_one_without( subject_id=StoreConstants.SUBJECT_ID_SAMPLE_WITH_ATTRIBUTES.value, invoice_id=StoreConstants.INVOICE_ID_SAMPLE_WITH_ATTRIBUTES.value, organism_id=StoreConstants.ORGANISM_ID_SAMPLE_WITH_ATTRIBUTES.value, - loqusdb_id=StoreConstants.LOCUSDB_ID_SAMPLE_WITH_ATTRIBUTES.value, + loqusdb_id=StoreConstants.LOQUSDB_ID_SAMPLE_WITH_ATTRIBUTES.value, downsampled_to=StoreConstants.DOWN_SAMPLED_TO_SAMPLE_WITH_ATTRIBUTES.value, no_invoice=False, age_at_sampling=StoreConstants.AGE_AT_SAMPLING_SAMPLE_WITH_ATTRIBUTES.value, diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py index 1fc4942c6e..70fd98efa1 100644 --- a/tests/utils/conftest.py +++ b/tests/utils/conftest.py @@ -25,7 +25,7 @@ def stderr_output(): lines = ( "2018-11-29 08:41:38 130-229-8-20-dhcp.local " "mongo_adapter.client[77135] INFO Connecting to " - "uri:mongodb://None:None@localhost:27017\n" + "uri:mongodb://None@localhost:27017\n" "2018-11-29 08:41:38 130-229-8-20-dhcp.local " "mongo_adapter.client[77135] INFO Connection " "established\n" From 75afb17b98f2388f7409dd9ee5c31db14271a4c6 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Mon, 20 May 2024 08:13:49 +0000 Subject: [PATCH 16/42] =?UTF-8?q?Bump=20version:=2060.7.25=20=E2=86=92=206?= =?UTF-8?q?0.7.26=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 5ba08a7671..3018fca3d4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.25 +current_version = 60.7.26 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 8c847c8248..1374b6bf5b 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.25" +__version__ = "60.7.26" diff --git a/pyproject.toml b/pyproject.toml index 8074259662..7b36cbe5e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.25" +version = "60.7.26" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 71217a152b0dc55393ac5b011667346d6c730d0f Mon Sep 17 00:00:00 2001 From: ChristianOertlin Date: Mon, 20 May 2024 12:27:25 +0200 Subject: [PATCH 17/42] refctor(sequencing metrics parser) (#3241) (patch) # Description Refactoring of sequencing metrics parser --- cg/apps/sequencing_metrics_parser/api.py | 76 ------------------ cg/meta/demultiplex/demux_post_processing.py | 6 +- .../status_db_storage_functions.py | 16 ++-- cg/meta/demultiplex/validation.py | 4 +- .../bcl_convert_metrics_service}/__init__.py | 0 .../bcl_convert_metrics_service.py | 79 +++++++++++++++++++ .../bcl_convert_metrics_service}/models.py | 0 .../bcl_convert_metrics_service}/parser.py | 8 +- tests/conftest.py | 8 ++ .../SampleSheet.csv | 0 .../Unaligned/Reports/Adapter_Metrics.csv | 0 .../Unaligned/Reports/Demultiplex_Stats.csv | 0 .../Unaligned/Reports/Quality_Metrics.csv | 0 .../__init__.py | 0 .../bcl_convert_metrics_service/__init__.py | 0 .../bcl_convert_metrics_service}/conftest.py | 14 +++- .../parsers/__init__.py | 0 .../parsers/test_bclconvert.py | 4 +- .../test_sequencing_metrics_parser_api.py | 25 +++--- 19 files changed, 137 insertions(+), 103 deletions(-) delete mode 100644 cg/apps/sequencing_metrics_parser/api.py rename cg/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/__init__.py (100%) create mode 100644 cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py rename cg/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/models.py (100%) rename cg/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/parser.py (97%) rename tests/fixtures/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/230622_A00621_0864_AHY7FFDRX2/SampleSheet.csv (100%) rename tests/fixtures/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Adapter_Metrics.csv (100%) rename tests/fixtures/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Demultiplex_Stats.csv (100%) rename tests/fixtures/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Quality_Metrics.csv (100%) rename tests/{apps/sequencing_metrics_parser => services}/__init__.py (100%) create mode 100644 tests/services/bcl_convert_metrics_service/__init__.py rename tests/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/conftest.py (84%) create mode 100644 tests/services/bcl_convert_metrics_service/parsers/__init__.py rename tests/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/parsers/test_bclconvert.py (97%) rename tests/{apps/sequencing_metrics_parser => services/bcl_convert_metrics_service}/parsers/test_sequencing_metrics_parser_api.py (74%) diff --git a/cg/apps/sequencing_metrics_parser/api.py b/cg/apps/sequencing_metrics_parser/api.py deleted file mode 100644 index 9c41be25d6..0000000000 --- a/cg/apps/sequencing_metrics_parser/api.py +++ /dev/null @@ -1,76 +0,0 @@ -from datetime import datetime -from pathlib import Path - -from cg.apps.sequencing_metrics_parser.parser import MetricsParser -from cg.constants.demultiplexing import UNDETERMINED -from cg.models.flow_cell.flow_cell import FlowCellDirectoryData -from cg.store.models import SampleLaneSequencingMetrics -from cg.utils.flow_cell import get_flow_cell_id - - -def create_sample_lane_sequencing_metrics_for_flow_cell( - flow_cell_directory: Path, -) -> list[SampleLaneSequencingMetrics]: - """Parse the demultiplexing metrics data into the sequencing statistics model.""" - metrics_parser = MetricsParser(flow_cell_directory) - sample_internal_ids: list[str] = metrics_parser.get_sample_internal_ids() - sample_lane_sequencing_metrics: list[SampleLaneSequencingMetrics] = [] - - for sample_internal_id in sample_internal_ids: - for lane in metrics_parser.get_lanes_for_sample(sample_internal_id=sample_internal_id): - metrics: SampleLaneSequencingMetrics = _create_bcl_convert_sequencing_metrics( - sample_internal_id=sample_internal_id, lane=lane, metrics_parser=metrics_parser - ) - sample_lane_sequencing_metrics.append(metrics) - return sample_lane_sequencing_metrics - - -def create_undetermined_non_pooled_metrics( - flow_cell: FlowCellDirectoryData, -) -> list[SampleLaneSequencingMetrics]: - """Return sequencing metrics for any undetermined reads in non-pooled lanes.""" - - non_pooled_lanes_and_samples: list[tuple[int, str]] = ( - flow_cell.sample_sheet.get_non_pooled_lanes_and_samples() - ) - metrics_parser = MetricsParser(flow_cell.path) - undetermined_metrics: list[SampleLaneSequencingMetrics] = [] - - for lane, sample_internal_id in non_pooled_lanes_and_samples: - if not metrics_parser.has_undetermined_reads_in_lane(lane): - continue - - # Passing Undetermined as the sample id is required to extract the undetermined reads data. - # BclConvert tags undetermined reads in a lane with the sample id "Undetermined". - metrics: SampleLaneSequencingMetrics = _create_bcl_convert_sequencing_metrics( - sample_internal_id=UNDETERMINED, lane=lane, metrics_parser=metrics_parser - ) - metrics.sample_internal_id = sample_internal_id - undetermined_metrics.append(metrics) - return undetermined_metrics - - -def _create_bcl_convert_sequencing_metrics( - sample_internal_id: str, lane: int, metrics_parser: MetricsParser -) -> SampleLaneSequencingMetrics: - """Create sequencing metrics for a sample in a lane.""" - flow_cell_id: str = get_flow_cell_id(metrics_parser.bcl_convert_demultiplex_dir.name) - - total_reads: int = metrics_parser.calculate_total_reads_for_sample_in_lane( - sample_internal_id=sample_internal_id, lane=lane - ) - q30_bases_percent: float = metrics_parser.get_q30_bases_percent_for_sample_in_lane( - sample_internal_id=sample_internal_id, lane=lane - ) - mean_quality_score: float = metrics_parser.get_mean_quality_score_for_sample_in_lane( - sample_internal_id=sample_internal_id, lane=lane - ) - return SampleLaneSequencingMetrics( - sample_internal_id=sample_internal_id, - flow_cell_name=flow_cell_id, - flow_cell_lane_number=lane, - sample_total_reads_in_lane=total_reads, - sample_base_percentage_passing_q30=q30_bases_percent, - sample_base_mean_quality_score=mean_quality_score, - created_at=datetime.now(), - ) diff --git a/cg/meta/demultiplex/demux_post_processing.py b/cg/meta/demultiplex/demux_post_processing.py index b3cd624491..120746a2f7 100644 --- a/cg/meta/demultiplex/demux_post_processing.py +++ b/cg/meta/demultiplex/demux_post_processing.py @@ -29,8 +29,10 @@ class DemuxPostProcessingAPI: def __init__(self, config: CGConfig) -> None: self.config: CGConfig = config - self.flow_cells_dir: Path = Path(config.illumina_flow_cells_directory) - self.demultiplexed_runs_dir: Path = Path(config.illumina_demultiplexed_runs_directory) + self.flow_cells_dir: Path = Path(config.run_instruments.illumina.flow_cell_runs_dir) + self.demultiplexed_runs_dir: Path = Path( + config.run_instruments.illumina.demultiplexed_runs_dir + ) self.status_db: Store = config.status_db self.hk_api: HousekeeperAPI = config.housekeeper_api self.dry_run: bool = False diff --git a/cg/meta/demultiplex/status_db_storage_functions.py b/cg/meta/demultiplex/status_db_storage_functions.py index 1536a0583c..5426f3840f 100644 --- a/cg/meta/demultiplex/status_db_storage_functions.py +++ b/cg/meta/demultiplex/status_db_storage_functions.py @@ -3,14 +3,15 @@ import datetime import logging -from cg.apps.sequencing_metrics_parser.api import ( - create_sample_lane_sequencing_metrics_for_flow_cell, - create_undetermined_non_pooled_metrics, -) + from cg.constants import FlowCellStatus from cg.meta.demultiplex.combine_sequencing_metrics import combine_mapped_metrics_with_undetermined from cg.meta.demultiplex.utils import get_q30_threshold from cg.models.flow_cell.flow_cell import FlowCellDirectoryData +from cg.services.bcl_convert_metrics_service.bcl_convert_metrics_service import ( + BCLConvertMetricsService, +) + from cg.store.models import Flowcell, Sample, SampleLaneSequencingMetrics from cg.store.store import Store @@ -43,11 +44,14 @@ def store_flow_cell_data_in_status_db( def store_sequencing_metrics_in_status_db(flow_cell: FlowCellDirectoryData, store: Store) -> None: + metrics_service = BCLConvertMetricsService() mapped_metrics: list[SampleLaneSequencingMetrics] = ( - create_sample_lane_sequencing_metrics_for_flow_cell(flow_cell_directory=flow_cell.path) + metrics_service.create_sample_lane_sequencing_metrics_for_flow_cell( + flow_cell_directory=flow_cell.path + ) ) undetermined_metrics: list[SampleLaneSequencingMetrics] = ( - create_undetermined_non_pooled_metrics(flow_cell) + metrics_service.create_undetermined_non_pooled_metrics(flow_cell) ) combined_metrics = combine_mapped_metrics_with_undetermined( diff --git a/cg/meta/demultiplex/validation.py b/cg/meta/demultiplex/validation.py index 067f4e0a9b..181a50b25e 100644 --- a/cg/meta/demultiplex/validation.py +++ b/cg/meta/demultiplex/validation.py @@ -52,7 +52,9 @@ def validate_flow_cell_has_fastq_files(flow_cell: FlowCellDirectoryData) -> None if fastq_files: LOG.debug(f"Flow cell {flow_cell.id} has at least one sample with fastq files") return - raise MissingFilesError(f"No fastq files were found for any sample in flow cell {flow_cell.id}") + raise MissingFilesError( + f"No fastq files were found for any sample in flow cell {flow_cell.id} path: {flow_cell.path}" + ) def is_flow_cell_ready_for_postprocessing( diff --git a/cg/apps/sequencing_metrics_parser/__init__.py b/cg/services/bcl_convert_metrics_service/__init__.py similarity index 100% rename from cg/apps/sequencing_metrics_parser/__init__.py rename to cg/services/bcl_convert_metrics_service/__init__.py diff --git a/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py b/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py new file mode 100644 index 0000000000..d2b3d87190 --- /dev/null +++ b/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py @@ -0,0 +1,79 @@ +from datetime import datetime +from pathlib import Path + + +from cg.constants.demultiplexing import UNDETERMINED +from cg.models.flow_cell.flow_cell import FlowCellDirectoryData +from cg.services.bcl_convert_metrics_service.parser import MetricsParser +from cg.store.models import SampleLaneSequencingMetrics +from cg.utils.flow_cell import get_flow_cell_id + + +class BCLConvertMetricsService: + def create_sample_lane_sequencing_metrics_for_flow_cell( + self, + flow_cell_directory: Path, + ) -> list[SampleLaneSequencingMetrics]: + """Parse the demultiplexing metrics data into the sequencing statistics model.""" + metrics_parser = MetricsParser(flow_cell_directory) + sample_internal_ids: list[str] = metrics_parser.get_sample_internal_ids() + sample_lane_sequencing_metrics: list[SampleLaneSequencingMetrics] = [] + + for sample_internal_id in sample_internal_ids: + for lane in metrics_parser.get_lanes_for_sample(sample_internal_id=sample_internal_id): + metrics: SampleLaneSequencingMetrics = self._create_bcl_convert_sequencing_metrics( + sample_internal_id=sample_internal_id, lane=lane, metrics_parser=metrics_parser + ) + sample_lane_sequencing_metrics.append(metrics) + return sample_lane_sequencing_metrics + + def create_undetermined_non_pooled_metrics( + self, + flow_cell: FlowCellDirectoryData, + ) -> list[SampleLaneSequencingMetrics]: + """Return sequencing metrics for any undetermined reads in non-pooled lanes.""" + + non_pooled_lanes_and_samples: list[tuple[int, str]] = ( + flow_cell.sample_sheet.get_non_pooled_lanes_and_samples() + ) + metrics_parser = MetricsParser(flow_cell.path) + undetermined_metrics: list[SampleLaneSequencingMetrics] = [] + + for lane, sample_internal_id in non_pooled_lanes_and_samples: + if not metrics_parser.has_undetermined_reads_in_lane(lane): + continue + + # Passing Undetermined as the sample id is required to extract the undetermined reads data. + # BclConvert tags undetermined reads in a lane with the sample id "Undetermined". + metrics: SampleLaneSequencingMetrics = self._create_bcl_convert_sequencing_metrics( + sample_internal_id=UNDETERMINED, lane=lane, metrics_parser=metrics_parser + ) + metrics.sample_internal_id = sample_internal_id + undetermined_metrics.append(metrics) + return undetermined_metrics + + @staticmethod + def _create_bcl_convert_sequencing_metrics( + sample_internal_id: str, lane: int, metrics_parser: MetricsParser + ) -> SampleLaneSequencingMetrics: + """Create sequencing metrics for a sample in a lane.""" + flow_cell_id: str = get_flow_cell_id(metrics_parser.bcl_convert_demultiplex_dir.name) + + total_reads: int = metrics_parser.calculate_total_reads_for_sample_in_lane( + sample_internal_id=sample_internal_id, lane=lane + ) + q30_bases_percent: float = metrics_parser.get_q30_bases_percent_for_sample_in_lane( + sample_internal_id=sample_internal_id, lane=lane + ) + mean_quality_score: float = metrics_parser.get_mean_quality_score_for_sample_in_lane( + sample_internal_id=sample_internal_id, lane=lane + ) + return SampleLaneSequencingMetrics( + sample_internal_id=sample_internal_id, + flow_cell_name=flow_cell_id, + flow_cell_lane_number=lane, + sample_total_reads_in_lane=total_reads, + sample_base_percentage_passing_q30=q30_bases_percent, + sample_base_mean_quality_score=mean_quality_score, + created_at=datetime.now(), + ) diff --git a/cg/apps/sequencing_metrics_parser/models.py b/cg/services/bcl_convert_metrics_service/models.py similarity index 100% rename from cg/apps/sequencing_metrics_parser/models.py rename to cg/services/bcl_convert_metrics_service/models.py diff --git a/cg/apps/sequencing_metrics_parser/parser.py b/cg/services/bcl_convert_metrics_service/parser.py similarity index 97% rename from cg/apps/sequencing_metrics_parser/parser.py rename to cg/services/bcl_convert_metrics_service/parser.py index bf1c53b049..a4f54bfd51 100644 --- a/cg/apps/sequencing_metrics_parser/parser.py +++ b/cg/services/bcl_convert_metrics_service/parser.py @@ -5,7 +5,7 @@ from typing import Callable from cg.apps.demultiplex.sample_sheet.validators import is_valid_sample_internal_id -from cg.apps.sequencing_metrics_parser.models import DemuxMetrics, SequencingQualityMetrics + from cg.constants.constants import SCALE_TO_READ_PAIRS, FileFormat from cg.constants.demultiplexing import UNDETERMINED from cg.constants.metrics import ( @@ -14,6 +14,7 @@ QUALITY_METRICS_FILE_NAME, ) from cg.io.controller import ReadFile +from cg.services.bcl_convert_metrics_service.models import DemuxMetrics, SequencingQualityMetrics from cg.utils.files import get_file_in_directory LOG = logging.getLogger(__name__) @@ -44,8 +45,9 @@ def __init__( metrics_model=DemuxMetrics, ) + @staticmethod def parse_metrics_file( - self, metrics_file_path, metrics_model: Callable + metrics_file_path, metrics_model: Callable ) -> list[SequencingQualityMetrics | DemuxMetrics]: """Parse specified metrics file.""" LOG.info(f"Parsing BCLConvert metrics file: {metrics_file_path}") @@ -75,8 +77,8 @@ def get_lanes_for_sample(self, sample_internal_id: str) -> list[int]: lanes_for_sample.append(sample_demux_metric.lane) return lanes_for_sample + @staticmethod def get_metrics_for_sample_and_lane( - self, metrics: list[SequencingQualityMetrics | DemuxMetrics], sample_internal_id: str, lane: int, diff --git a/tests/conftest.py b/tests/conftest.py index e12ed3f5fc..919b0ff67f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -63,6 +63,9 @@ TaxprofilerSampleSheetEntry, ) from cg.models.tomte.tomte import TomteParameters, TomteSampleSheetHeaders +from cg.services.bcl_convert_metrics_service.bcl_convert_metrics_service import ( + BCLConvertMetricsService, +) from cg.store.database import create_all_tables, drop_all_tables, initialize_database from cg.store.models import Bed, BedVersion, Case, Customer, Order, Organism, Sample from cg.store.store import Store @@ -3955,3 +3958,8 @@ def fastq_file_meta_raw(flow_cell_name: str) -> dict: "flow_cell_id": flow_cell_name, "undetermined": None, } + + +@pytest.fixture() +def bcl_convert_metrics_service() -> BCLConvertMetricsService: + return BCLConvertMetricsService() diff --git a/tests/fixtures/apps/sequencing_metrics_parser/230622_A00621_0864_AHY7FFDRX2/SampleSheet.csv b/tests/fixtures/services/bcl_convert_metrics_service/230622_A00621_0864_AHY7FFDRX2/SampleSheet.csv similarity index 100% rename from tests/fixtures/apps/sequencing_metrics_parser/230622_A00621_0864_AHY7FFDRX2/SampleSheet.csv rename to tests/fixtures/services/bcl_convert_metrics_service/230622_A00621_0864_AHY7FFDRX2/SampleSheet.csv diff --git a/tests/fixtures/apps/sequencing_metrics_parser/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Adapter_Metrics.csv b/tests/fixtures/services/bcl_convert_metrics_service/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Adapter_Metrics.csv similarity index 100% rename from tests/fixtures/apps/sequencing_metrics_parser/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Adapter_Metrics.csv rename to tests/fixtures/services/bcl_convert_metrics_service/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Adapter_Metrics.csv diff --git a/tests/fixtures/apps/sequencing_metrics_parser/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Demultiplex_Stats.csv b/tests/fixtures/services/bcl_convert_metrics_service/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Demultiplex_Stats.csv similarity index 100% rename from tests/fixtures/apps/sequencing_metrics_parser/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Demultiplex_Stats.csv rename to tests/fixtures/services/bcl_convert_metrics_service/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Demultiplex_Stats.csv diff --git a/tests/fixtures/apps/sequencing_metrics_parser/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Quality_Metrics.csv b/tests/fixtures/services/bcl_convert_metrics_service/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Quality_Metrics.csv similarity index 100% rename from tests/fixtures/apps/sequencing_metrics_parser/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Quality_Metrics.csv rename to tests/fixtures/services/bcl_convert_metrics_service/230622_A00621_0864_AHY7FFDRX2/Unaligned/Reports/Quality_Metrics.csv diff --git a/tests/apps/sequencing_metrics_parser/__init__.py b/tests/services/__init__.py similarity index 100% rename from tests/apps/sequencing_metrics_parser/__init__.py rename to tests/services/__init__.py diff --git a/tests/services/bcl_convert_metrics_service/__init__.py b/tests/services/bcl_convert_metrics_service/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/apps/sequencing_metrics_parser/conftest.py b/tests/services/bcl_convert_metrics_service/conftest.py similarity index 84% rename from tests/apps/sequencing_metrics_parser/conftest.py rename to tests/services/bcl_convert_metrics_service/conftest.py index d5ee48cd0a..a73ac3e763 100644 --- a/tests/apps/sequencing_metrics_parser/conftest.py +++ b/tests/services/bcl_convert_metrics_service/conftest.py @@ -4,16 +4,24 @@ import pytest -from cg.apps.sequencing_metrics_parser.models import DemuxMetrics, SequencingQualityMetrics -from cg.apps.sequencing_metrics_parser.parser import MetricsParser + from cg.constants.metrics import DemuxMetricsColumnNames, QualityMetricsColumnNames +from cg.services.bcl_convert_metrics_service.bcl_convert_metrics_service import ( + BCLConvertMetricsService, +) +from cg.services.bcl_convert_metrics_service.models import DemuxMetrics, SequencingQualityMetrics +from cg.services.bcl_convert_metrics_service.parser import MetricsParser @pytest.fixture(scope="session") def bcl_convert_metrics_dir_path() -> Path: """Return a path to a BCLConvert metrics directory.""" return Path( - "tests", "fixtures", "apps", "sequencing_metrics_parser", "230622_A00621_0864_AHY7FFDRX2" + "tests", + "fixtures", + "services", + "bcl_convert_metrics_service", + "230622_A00621_0864_AHY7FFDRX2", ) diff --git a/tests/services/bcl_convert_metrics_service/parsers/__init__.py b/tests/services/bcl_convert_metrics_service/parsers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/apps/sequencing_metrics_parser/parsers/test_bclconvert.py b/tests/services/bcl_convert_metrics_service/parsers/test_bclconvert.py similarity index 97% rename from tests/apps/sequencing_metrics_parser/parsers/test_bclconvert.py rename to tests/services/bcl_convert_metrics_service/parsers/test_bclconvert.py index a9d825f5ad..e5ed099b95 100644 --- a/tests/apps/sequencing_metrics_parser/parsers/test_bclconvert.py +++ b/tests/services/bcl_convert_metrics_service/parsers/test_bclconvert.py @@ -4,8 +4,8 @@ import pytest -from cg.apps.sequencing_metrics_parser.models import DemuxMetrics, SequencingQualityMetrics -from cg.apps.sequencing_metrics_parser.parser import MetricsParser +from cg.services.bcl_convert_metrics_service.models import SequencingQualityMetrics, DemuxMetrics +from cg.services.bcl_convert_metrics_service.parser import MetricsParser def test_parse_metrics( diff --git a/tests/apps/sequencing_metrics_parser/parsers/test_sequencing_metrics_parser_api.py b/tests/services/bcl_convert_metrics_service/parsers/test_sequencing_metrics_parser_api.py similarity index 74% rename from tests/apps/sequencing_metrics_parser/parsers/test_sequencing_metrics_parser_api.py rename to tests/services/bcl_convert_metrics_service/parsers/test_sequencing_metrics_parser_api.py index ecf2726a06..4df1db2a40 100644 --- a/tests/apps/sequencing_metrics_parser/parsers/test_sequencing_metrics_parser_api.py +++ b/tests/services/bcl_convert_metrics_service/parsers/test_sequencing_metrics_parser_api.py @@ -2,25 +2,26 @@ from pathlib import Path -from cg.apps.sequencing_metrics_parser.api import ( - create_sample_lane_sequencing_metrics_for_flow_cell, - create_undetermined_non_pooled_metrics, -) -from cg.apps.sequencing_metrics_parser.parser import MetricsParser from cg.models.flow_cell.flow_cell import FlowCellDirectoryData +from cg.services.bcl_convert_metrics_service.bcl_convert_metrics_service import ( + BCLConvertMetricsService, +) + +from cg.services.bcl_convert_metrics_service.parser import MetricsParser from cg.store.models import SampleLaneSequencingMetrics def test_create_sample_lane_sequencing_metrics_for_flow_cell( bcl_convert_metrics_dir_path: Path, parsed_bcl_convert_metrics: MetricsParser, + bcl_convert_metrics_service: BCLConvertMetricsService, ): """Test to create sequencing statistics from bcl convert metrics.""" # GIVEN a parsed bcl convert metrics file # WHEN creating sequencing statistics from bcl convert metrics sequencing_statistics_list: list[SampleLaneSequencingMetrics] = ( - create_sample_lane_sequencing_metrics_for_flow_cell( + bcl_convert_metrics_service.create_sample_lane_sequencing_metrics_for_flow_cell( flow_cell_directory=bcl_convert_metrics_dir_path, ) ) @@ -38,13 +39,16 @@ def test_create_sample_lane_sequencing_metrics_for_flow_cell( def test_create_undetermined_non_pooled_metrics( hiseq_x_single_index_demultiplexed_flow_cell_with_sample_sheet: FlowCellDirectoryData, + bcl_convert_metrics_service: BCLConvertMetricsService, ): """Test creating undetermined sequencing statistics from demultiplex metrics.""" # GIVEN a directory with a demultiplexed flow cell with undetermined reads # WHEN creating undetermined sequencing statistics from bcl convert metrics - metrics: list[SampleLaneSequencingMetrics] = create_undetermined_non_pooled_metrics( - flow_cell=hiseq_x_single_index_demultiplexed_flow_cell_with_sample_sheet + metrics: list[SampleLaneSequencingMetrics] = ( + bcl_convert_metrics_service.create_undetermined_non_pooled_metrics( + flow_cell=hiseq_x_single_index_demultiplexed_flow_cell_with_sample_sheet + ) ) # THEN metrics are created for the undetermined reads @@ -54,6 +58,7 @@ def test_create_undetermined_non_pooled_metrics( def test_create_undetermined_non_pooled_metrics_for_existing_lane_without_undetermined_reads( bcl_convert_metrics_dir_path: Path, + bcl_convert_metrics_service: BCLConvertMetricsService, ): """ Test creating undetermined sequencing statistics from demultiplex metrics without undetermined @@ -66,8 +71,8 @@ def test_create_undetermined_non_pooled_metrics_for_existing_lane_without_undete flow_cell.set_sample_sheet_path_hk(hk_path=sample_sheet_path) # WHEN creating undetermined sequencing statistics specifying a lane without undetermined reads - metrics: list[SampleLaneSequencingMetrics] = create_undetermined_non_pooled_metrics( - flow_cell=flow_cell + metrics: list[SampleLaneSequencingMetrics] = ( + bcl_convert_metrics_service.create_undetermined_non_pooled_metrics(flow_cell=flow_cell) ) # THEN an empty list is returned From ec1b55ec94087c3f44e1f844ad7dec6fb88109bb Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Mon, 20 May 2024 10:27:54 +0000 Subject: [PATCH 18/42] =?UTF-8?q?Bump=20version:=2060.7.26=20=E2=86=92=206?= =?UTF-8?q?0.7.27=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3018fca3d4..d288dfd509 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.26 +current_version = 60.7.27 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 1374b6bf5b..96e30a0f7c 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.26" +__version__ = "60.7.27" diff --git a/pyproject.toml b/pyproject.toml index 7b36cbe5e3..d9c098fadb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.26" +version = "60.7.27" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From df084c3c8326e78c2d21f8be3326a4b1c301a455 Mon Sep 17 00:00:00 2001 From: ChristianOertlin Date: Mon, 20 May 2024 13:45:33 +0200 Subject: [PATCH 19/42] remove dead code (#3248)(patch) # description removes unused code --- cg/meta/demultiplex/files.py | 93 ------------------------------------ 1 file changed, 93 deletions(-) delete mode 100644 cg/meta/demultiplex/files.py diff --git a/cg/meta/demultiplex/files.py b/cg/meta/demultiplex/files.py deleted file mode 100644 index c209a3b2ef..0000000000 --- a/cg/meta/demultiplex/files.py +++ /dev/null @@ -1,93 +0,0 @@ -import logging -import sys -from pathlib import Path - -from cg.constants.demultiplexing import FASTQ_FILE_SUFFIXES -from cg.constants.symbols import UNDERSCORE - -LOG = logging.getLogger(__name__) - - -def rename_project_directory( - project_directory: Path, flow_cell_id: str, dry_run: bool = False -) -> None: - """Rename a project directory by adding the prefix Project_.""" - unaligned_directory: Path = project_directory.parent - LOG.info("Check for Dragen fastq files in project directories.") - if dragen_fastq_files_in_project_directory(project_directory): - LOG.debug("Only Dragen fastq files found!") - move_dragen_fastq_files(project_directory=project_directory, dry_run=dry_run) - if dry_run: - LOG.info("Can't continue dry run...") - sys.exit() - - LOG.debug(f"Rename all sample directories in {unaligned_directory}") - for sample_dir in project_directory.iterdir(): - if sample_dir.name.startswith("Sample"): - LOG.debug(f"{sample_dir} is already renamed") - continue - rename_sample_directory( - sample_directory=sample_dir, flow_cell_id=flow_cell_id, dry_run=dry_run - ) - new_name: str = "_".join(["Project", project_directory.name]) - new_project_path: Path = Path(unaligned_directory, new_name) - LOG.debug(f"Rename project dir {project_directory}") - if not dry_run: - LOG.debug(f"Rename project dir to {new_project_path}") - project_directory.rename(new_project_path) - - -def move_dragen_fastq_files(project_directory: Path, dry_run: bool = False) -> None: - """Move Dragen fastq files into sample directories""" - - for dragen_fastq_file in project_directory.iterdir(): - LOG.debug( - f"Derive sample name from fastq file {dragen_fastq_file}: {get_dragen_sample_name(dragen_fastq_file)}", - ) - dragen_sample_name: str = get_dragen_sample_name(dragen_fastq_file) - LOG.debug(f"Create sample directory {project_directory / dragen_sample_name}:") - if not dry_run: - (project_directory / dragen_sample_name).mkdir(exist_ok=True) - target_directory: Path = project_directory / dragen_sample_name / dragen_fastq_file.name - LOG.debug(f"Move fastq file into sample directory: {target_directory}") - if not dry_run: - dragen_fastq_file.rename(target_directory) - - -def rename_sample_directory( - sample_directory: Path, flow_cell_id: str, dry_run: bool = False -) -> None: - """Rename a sample dir and all fastq files in the sample dir. - - Renaming of the sample dir means adding Sample_ as a prefix. - """ - project_directory: Path = sample_directory.parent - LOG.debug(f"Renaming all fastq files in {sample_directory}") - for fastq_file in sample_directory.iterdir(): - rename_fastq_file(fastq_file=fastq_file, flow_cell_id=flow_cell_id, dry_run=dry_run) - new_name: str = "_".join(["Sample", sample_directory.name]) - new_sample_directory: Path = Path(project_directory, new_name) - LOG.debug(f"Renaming sample dir {sample_directory} to {new_sample_directory}") - if not dry_run: - sample_directory.rename(new_sample_directory) - LOG.debug(f"Renamed sample dir to {new_sample_directory}") - - -def rename_fastq_file(fastq_file: Path, flow_cell_id: str, dry_run: bool = False) -> None: - """Rename a fastq file by appending the flowcell id as a prefix.""" - new_name: str = "_".join([flow_cell_id, fastq_file.name]) - new_file: Path = Path(fastq_file.parent, new_name) - LOG.debug(f"Renaming fastq file {fastq_file} to {new_file}") - if not dry_run: - fastq_file.rename(new_file) - LOG.debug(f"Renamed fastq file to {new_file}") - - -def dragen_fastq_files_in_project_directory(project_directory: Path) -> bool: - """Checks if the project directory contains Dragen fastq files instead of sample directories""" - return all(file_.suffixes == FASTQ_FILE_SUFFIXES for file_ in project_directory.iterdir()) - - -def get_dragen_sample_name(dragen_fastq_file: Path) -> str: - """Derives the sample name from a dragen fastq file""" - return dragen_fastq_file.name.split("_")[0] From aa24b27ea297a5db7de73902e8d2e37ecc78ac31 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Mon, 20 May 2024 11:45:59 +0000 Subject: [PATCH 20/42] =?UTF-8?q?Bump=20version:=2060.7.27=20=E2=86=92=206?= =?UTF-8?q?0.7.28=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d288dfd509..e6e733bdf5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.27 +current_version = 60.7.28 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 96e30a0f7c..3e5e61d01b 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.27" +__version__ = "60.7.28" diff --git a/pyproject.toml b/pyproject.toml index d9c098fadb..db2ec3dab1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.27" +version = "60.7.28" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 629fe6bb2ab6291d26f439f8e4143484e7384b37 Mon Sep 17 00:00:00 2001 From: Vincent Janvid <69356202+Vince-janv@users.noreply.github.com> Date: Mon, 20 May 2024 14:12:19 +0200 Subject: [PATCH 21/42] feat(Github action) Make Pytest use 8-core runner (#3242)(patch) ## Description Changes the pytest github action runner to an 8-core one ### Changed - pytest action runs on 8-core machins --- .github/workflows/tests_and_coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_and_coverage.yml b/.github/workflows/tests_and_coverage.yml index cf9e77110f..2b009111c9 100644 --- a/.github/workflows/tests_and_coverage.yml +++ b/.github/workflows/tests_and_coverage.yml @@ -8,7 +8,7 @@ on: jobs: tests-coverage: - runs-on: Beefy_Linux + runs-on: 8_core_linux steps: - name: Checkout Repository From b4cbef5803f6df9c5feeae1c05d028c30d9521cb Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Mon, 20 May 2024 12:12:48 +0000 Subject: [PATCH 22/42] =?UTF-8?q?Bump=20version:=2060.7.28=20=E2=86=92=206?= =?UTF-8?q?0.7.29=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e6e733bdf5..c2c050371a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.28 +current_version = 60.7.29 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 3e5e61d01b..f5c15448b7 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.28" +__version__ = "60.7.29" diff --git a/pyproject.toml b/pyproject.toml index db2ec3dab1..1fcfd43e63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.28" +version = "60.7.29" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From d10ab8520d4385dcdbbdc698d69339305cfed889 Mon Sep 17 00:00:00 2001 From: Beatriz Vinhas Date: Mon, 20 May 2024 15:45:32 +0200 Subject: [PATCH 23/42] Remove obsolete panels from the DSD Scout bundle (#3247)(patch) ### Changed - Remove obsolete panels SEXDET, SEXDIF from the DSD Scout bundle --- cg/constants/gene_panel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cg/constants/gene_panel.py b/cg/constants/gene_panel.py index 81bba3c87a..4ad78c87b8 100644 --- a/cg/constants/gene_panel.py +++ b/cg/constants/gene_panel.py @@ -67,7 +67,7 @@ def collaborators() -> set[str]: class GenePanelCombo: COMBO_1: dict[str, set[str]] = { - "DSD": {"DSD", "DSD-S", "HYP", "SEXDIF", "SEXDET", "POI"}, + "DSD": {"DSD", "DSD-S", "HYP", "POI"}, "CM": {"CNM", "CM"}, "Horsel": {"Horsel", "141217", "141201"}, "OPHTHALMO": { From cfc76655889bb68e1f4ca753ee2cdbc617187124 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Mon, 20 May 2024 13:45:59 +0000 Subject: [PATCH 24/42] =?UTF-8?q?Bump=20version:=2060.7.29=20=E2=86=92=206?= =?UTF-8?q?0.7.30=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c2c050371a..978da9762e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.29 +current_version = 60.7.30 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index f5c15448b7..ef1ee43ac3 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.29" +__version__ = "60.7.30" diff --git a/pyproject.toml b/pyproject.toml index 1fcfd43e63..0bdbfca6d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.29" +version = "60.7.30" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 2c52af4ab200f62753ce2c2aebe133011bca0999 Mon Sep 17 00:00:00 2001 From: Beatriz Vinhas Date: Tue, 21 May 2024 14:28:46 +0200 Subject: [PATCH 25/42] Include multiqc_report for mip-dna delivery (#3259)(patch) ### Added - "multiqc-html" tag to "MIP_DNA_ANALYSIS_CASE_TAGS. --- cg/constants/delivery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cg/constants/delivery.py b/cg/constants/delivery.py index 75f3425358..5368a11fbe 100644 --- a/cg/constants/delivery.py +++ b/cg/constants/delivery.py @@ -78,6 +78,7 @@ MIP_DNA_ANALYSIS_CASE_TAGS: list[set[str]] = [ {"delivery-report"}, + {"multiqc-html"}, {"vcf-clinical-sv-bin"}, {"vcf-clinical-sv-bin-index"}, {"vcf-research-sv-bin"}, From 5b2f42cd813e901cf8e63d613877401cd5669ae3 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Tue, 21 May 2024 12:29:13 +0000 Subject: [PATCH 26/42] =?UTF-8?q?Bump=20version:=2060.7.30=20=E2=86=92=206?= =?UTF-8?q?0.7.31=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 978da9762e..06e97da89a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.30 +current_version = 60.7.31 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index ef1ee43ac3..ea7ff348df 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.30" +__version__ = "60.7.31" diff --git a/pyproject.toml b/pyproject.toml index 0bdbfca6d0..365fdd4f90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.30" +version = "60.7.31" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 10c2025df2334b1c5b69adc3dd365f98985acb8c Mon Sep 17 00:00:00 2001 From: Vincent Janvid <69356202+Vince-janv@users.noreply.github.com> Date: Wed, 22 May 2024 08:48:27 +0200 Subject: [PATCH 27/42] Feat(Sequencing models) Add relationships (#3246)(minor) ## Description With the models in place relationships can be defined to related them in python. Also two basic properties where added. ### Added - Relationships between device -> run_metrics -> sample_run_metrics - property to access devices from samples - property to access samples from device --- cg/store/models.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cg/store/models.py b/cg/store/models.py index 9f05b7b314..0f2475b2cd 100644 --- a/cg/store/models.py +++ b/cg/store/models.py @@ -782,6 +782,10 @@ class Sample(Base, PriorityMixin): ) invoice: Mapped["Invoice | None"] = orm.relationship(back_populates="samples") + _new_run_metrics: Mapped[list["SampleRunMetrics"]] = orm.relationship( + back_populates="sample", cascade="all, delete" + ) + def __str__(self) -> str: return f"{self.internal_id} ({self.name})" @@ -839,6 +843,11 @@ def state(self) -> str: return f"Ordered {self.ordered_at.date()}" + @property + def _run_devices(self) -> list["RunDevice"]: + """Return the devices a sample has been sequenced on.""" + return list({metric.run_metrics.device for metric in self._new_run_metrics}) + def to_dict(self, links: bool = False, flowcells: bool = False) -> dict: """Represent as dictionary""" data = to_dict(model_instance=self) @@ -960,6 +969,21 @@ class RunDevice(Base): type: Mapped[DeviceType] internal_id: Mapped[UniqueStr64] + run_metrics: Mapped[list["RunMetrics"]] = orm.relationship( + back_populates="device", cascade="all, delete" + ) + + @property + def _samples(self) -> list[Sample]: + """Return the samples sequenced in this device.""" + return list( + { + sample_run_metric.sample + for run in self.run_metrics + for sample_run_metric in run.sample_run_metrics + } + ) + __mapper_args__ = { "polymorphic_on": "type", } @@ -987,6 +1011,11 @@ class RunMetrics(Base): type: Mapped[DeviceType] device_id: Mapped[int] = mapped_column(ForeignKey("run_device.id")) + device: Mapped[RunDevice] = orm.relationship(back_populates="run_metrics") + sample_metrics: Mapped[list["SampleRunMetrics"]] = orm.relationship( + back_populates="run_metrics", cascade="all, delete" + ) + __mapper_args__ = { "polymorphic_on": "type", } @@ -1030,6 +1059,9 @@ class SampleRunMetrics(Base): run_metrics_id: Mapped[int] = mapped_column(ForeignKey("run_metrics.id")) type: Mapped[DeviceType] + run_metrics: Mapped[RunMetrics] = orm.relationship(back_populates="sample_metrics") + sample: Mapped[Sample] = orm.relationship(back_populates="_new_run_metrics") + __mapper_args__ = { "polymorphic_on": "type", } From 27d967df0ba75d38e4e7002319d233e0f298205f Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Wed, 22 May 2024 06:48:55 +0000 Subject: [PATCH 28/42] =?UTF-8?q?Bump=20version:=2060.7.31=20=E2=86=92=206?= =?UTF-8?q?0.8.0=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 06e97da89a..021fe217f1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.7.31 +current_version = 60.8.0 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index ea7ff348df..2fba876816 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.7.31" +__version__ = "60.8.0" diff --git a/pyproject.toml b/pyproject.toml index 365fdd4f90..0a9813f540 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.7.31" +version = "60.8.0" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 2ab0791c180e4087a020f5d4765b6c6e62f60b27 Mon Sep 17 00:00:00 2001 From: Sebastian Diaz Date: Wed, 22 May 2024 09:17:44 +0200 Subject: [PATCH 29/42] add autoincrement for run device (#3257)(patch) ## Description This PR does 3 fixes to the database run tables: 1. Modifies the primary keys of `run_device`, `run_metrics` and `sample_run_metrics` models in an alembic revision to include the autoincrement to the key values. For the primary keys to be modified, their use as foreign keys has to be temporarily suspended, that is why there is a drop constraint command before the modification and a creation of a constraint after. 2. The column `model` from `illumina_flow_cell` is set to nullable. 3. The column `run_metrics_id` in the `sample_run_metrics` didn't have its foreign key set in alembic, so now the constraint is added. --- ...8fb1_add_autoincrement_to_device_tables.py | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 alembic/versions/5fd7e8758fb1_add_autoincrement_to_device_tables.py diff --git a/alembic/versions/5fd7e8758fb1_add_autoincrement_to_device_tables.py b/alembic/versions/5fd7e8758fb1_add_autoincrement_to_device_tables.py new file mode 100644 index 0000000000..2c9ee1d55c --- /dev/null +++ b/alembic/versions/5fd7e8758fb1_add_autoincrement_to_device_tables.py @@ -0,0 +1,143 @@ +"""add_autoincrement_to_device_tables + +Revision ID: 5fd7e8758fb1 +Revises: fe23de4ed528 +Create Date: 2024-05-21 09:25:47.751986 + +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "5fd7e8758fb1" +down_revision = "fe23de4ed528" +branch_labels = None +depends_on = None + + +def upgrade(): + # Modify illumina flow cell primary key + op.drop_constraint( + constraint_name="illumina_flow_cell_ibfk_1", + table_name="illumina_flow_cell", + type_="foreignkey", + ) + op.drop_constraint(constraint_name="fk_device_id", table_name="run_metrics", type_="foreignkey") + op.execute("ALTER TABLE run_device MODIFY COLUMN id INT NOT NULL AUTO_INCREMENT") + op.alter_column( + "illumina_flow_cell", "model", existing_type=sa.String(length=32), nullable=True + ) + op.create_foreign_key( + constraint_name="fk_device_id", + source_table="run_metrics", + referent_table="run_device", + local_cols=["device_id"], + remote_cols=["id"], + ) + op.create_foreign_key( + constraint_name="illumina_flow_cell_ibfk_1", + source_table="illumina_flow_cell", + referent_table="run_device", + local_cols=["id"], + remote_cols=["id"], + ) + + # Modify run metrics primary key + op.drop_constraint( + constraint_name="illumina_sequencing_metrics_ibfk_1", + table_name="illumina_sequencing_metrics", + type_="foreignkey", + ) + op.execute("ALTER TABLE run_metrics MODIFY COLUMN id INT NOT NULL AUTO_INCREMENT") + op.create_foreign_key( + constraint_name="illumina_sequencing_metrics_ibfk_1", + source_table="illumina_sequencing_metrics", + referent_table="run_metrics", + local_cols=["id"], + remote_cols=["id"], + ) + op.create_foreign_key( + constraint_name="fk_run_metrics_id", + source_table="sample_run_metrics", + referent_table="run_metrics", + local_cols=["run_metrics_id"], + remote_cols=["id"], + ) + + # Modify sample run metrics primary key + op.drop_constraint( + constraint_name="illumina_sample_run_metrics_ibfk_1", + table_name="illumina_sample_run_metrics", + type_="foreignkey", + ) + op.execute("ALTER TABLE sample_run_metrics MODIFY COLUMN id INT NOT NULL AUTO_INCREMENT") + op.create_foreign_key( + constraint_name="illumina_sample_run_metrics_ibfk_1", + source_table="illumina_sample_run_metrics", + referent_table="sample_run_metrics", + local_cols=["id"], + remote_cols=["id"], + ) + + +def downgrade(): + # Downgrade illumina flow cell primary key + op.drop_constraint( + constraint_name="illumina_flow_cell_ibfk_1", + table_name="illumina_flow_cell", + type_="foreignkey", + ) + op.drop_constraint(constraint_name="fk_device_id", table_name="run_metrics", type_="foreignkey") + op.alter_column( + "illumina_flow_cell", "model", existing_type=sa.String(length=32), nullable=False + ) + op.execute("ALTER TABLE run_device MODIFY COLUMN id INT NOT NULL") + op.create_foreign_key( + constraint_name="fk_device_id", + source_table="run_metrics", + referent_table="run_device", + local_cols=["device_id"], + remote_cols=["id"], + ) + op.create_foreign_key( + constraint_name="illumina_flow_cell_ibfk_1", + source_table="illumina_flow_cell", + referent_table="run_device", + local_cols=["id"], + remote_cols=["id"], + ) + + # Downgrade illumina run metrics primary key + op.drop_constraint( + constraint_name="fk_run_metrics_id", table_name="sample_run_metrics", type_="foreignkey" + ) + op.drop_constraint( + constraint_name="illumina_sequencing_metrics_ibfk_1", + table_name="illumina_sequencing_metrics", + type_="foreignkey", + ) + op.execute("ALTER TABLE run_metrics MODIFY COLUMN id INT NOT NULL") + op.create_foreign_key( + constraint_name="illumina_sequencing_metrics_ibfk_1", + source_table="illumina_sequencing_metrics", + referent_table="run_metrics", + local_cols=["id"], + remote_cols=["id"], + ) + + # Downgrade illumina run sample metrics primary key + op.drop_constraint( + constraint_name="illumina_sample_run_metrics_ibfk_1", + table_name="illumina_sample_run_metrics", + type_="foreignkey", + ) + op.execute("ALTER TABLE sample_run_metrics MODIFY COLUMN id INT NOT NULL") + op.create_foreign_key( + constraint_name="illumina_sample_run_metrics_ibfk_1", + source_table="illumina_sample_run_metrics", + referent_table="sample_run_metrics", + local_cols=["id"], + remote_cols=["id"], + ) From 80158d0031424b719fb119aff9f2753fe40aa97c Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Wed, 22 May 2024 07:18:10 +0000 Subject: [PATCH 30/42] =?UTF-8?q?Bump=20version:=2060.8.0=20=E2=86=92=2060?= =?UTF-8?q?.8.1=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 021fe217f1..c17778e6ed 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.8.0 +current_version = 60.8.1 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 2fba876816..d858f49281 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.8.0" +__version__ = "60.8.1" diff --git a/pyproject.toml b/pyproject.toml index 0a9813f540..57d73fce14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.8.0" +version = "60.8.1" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 5e086308e990a740ca6df83db557a587ccae9872 Mon Sep 17 00:00:00 2001 From: ChristianOertlin Date: Wed, 22 May 2024 09:44:15 +0200 Subject: [PATCH 31/42] refactor(demux flow for new flow cell model pt2) (#3256)(patch) # Description Adds functionality to rewire the demux post processing flow to use the new tables part2. Added a service IlluminaPostProcessingService that will hold the new logic Added store functions to add and retrieve an IlluminaFlowCell from the store. Moved demultiplex post processing validation module to the new service and changed imports related to this move --- cg/constants/demultiplexing.py | 2 + cg/meta/demultiplex/demux_post_processing.py | 4 +- cg/models/flow_cell/flow_cell.py | 2 - .../bcl_convert_metrics_service.py | 36 +++++- .../__init__.py | 0 .../illumina_post_processing_service.py | 103 ++++++++++++++++++ .../illumina_post_processing_service/utils.py | 25 +++++ .../validation.py | 0 cg/store/crud/create.py | 10 ++ cg/store/crud/read.py | 13 +++ .../status_illumina_flow_cell_filters.py | 32 ++++++ cg/store/store.py | 4 + tests/meta/demultiplex/test_validation.py | 2 +- .../conftest.py | 31 ++++++ .../test_illumina_post_processing_utils.py | 43 ++++++++ tests/store/conftest.py | 23 ++++ tests/store/crud/add/test_store_add_base.py | 32 ++++++ tests/store/crud/conftest.py | 2 - .../crud/read/test_read_illumina_flow_cell.py | 20 ++++ .../test_status_illumina_flow_cell_filters.py | 27 +++++ 20 files changed, 404 insertions(+), 7 deletions(-) create mode 100644 cg/services/illumina_post_processing_service/__init__.py create mode 100644 cg/services/illumina_post_processing_service/illumina_post_processing_service.py create mode 100644 cg/services/illumina_post_processing_service/utils.py rename cg/{meta/demultiplex => services/illumina_post_processing_service}/validation.py (100%) create mode 100644 cg/store/filters/status_illumina_flow_cell_filters.py create mode 100644 tests/services/illumina_post_processing_service/conftest.py create mode 100644 tests/services/illumina_post_processing_service/test_illumina_post_processing_utils.py create mode 100644 tests/store/crud/read/test_read_illumina_flow_cell.py create mode 100644 tests/store/filters/test_status_illumina_flow_cell_filters.py diff --git a/cg/constants/demultiplexing.py b/cg/constants/demultiplexing.py index aec437e6d0..bb301e93ad 100644 --- a/cg/constants/demultiplexing.py +++ b/cg/constants/demultiplexing.py @@ -57,6 +57,8 @@ class RunParametersXMLNodes(StrEnum): READ_NAME: str = "ReadName" REAGENT_KIT_VERSION: str = "./RfidsInfo/SbsConsumableVersion" SEQUENCER_ID: str = ".//ScannerID" + FLOW_CELL_MODE: str = ".//FlowCellMode" + MODE: str = ".//Mode" # Node Values HISEQ_APPLICATION: str = "HiSeq Control Software" diff --git a/cg/meta/demultiplex/demux_post_processing.py b/cg/meta/demultiplex/demux_post_processing.py index 120746a2f7..5f56cc0cce 100644 --- a/cg/meta/demultiplex/demux_post_processing.py +++ b/cg/meta/demultiplex/demux_post_processing.py @@ -16,7 +16,9 @@ store_sequencing_metrics_in_status_db, ) from cg.meta.demultiplex.utils import create_delivery_file_in_flow_cell_directory -from cg.meta.demultiplex.validation import is_flow_cell_ready_for_postprocessing +from cg.services.illumina_post_processing_service.validation import ( + is_flow_cell_ready_for_postprocessing, +) from cg.models.cg_config import CGConfig from cg.models.flow_cell.flow_cell import FlowCellDirectoryData from cg.store.store import Store diff --git a/cg/models/flow_cell/flow_cell.py b/cg/models/flow_cell/flow_cell.py index 761cb3e8c2..7e256a1c3d 100644 --- a/cg/models/flow_cell/flow_cell.py +++ b/cg/models/flow_cell/flow_cell.py @@ -5,9 +5,7 @@ import os from pathlib import Path from typing import Type - from typing_extensions import Literal - from cg.apps.demultiplex.sample_sheet.sample_sheet_models import SampleSheet from cg.apps.demultiplex.sample_sheet.sample_sheet_validator import SampleSheetValidator from cg.cli.demultiplex.copy_novaseqx_demultiplex_data import get_latest_analysis_path diff --git a/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py b/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py index d2b3d87190..c18780b7be 100644 --- a/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py +++ b/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py @@ -3,9 +3,11 @@ from cg.constants.demultiplexing import UNDETERMINED +from cg.constants.devices import DeviceType from cg.models.flow_cell.flow_cell import FlowCellDirectoryData from cg.services.bcl_convert_metrics_service.parser import MetricsParser -from cg.store.models import SampleLaneSequencingMetrics +from cg.store.models import SampleLaneSequencingMetrics, IlluminaSampleRunMetrics +from cg.store.store import Store from cg.utils.flow_cell import get_flow_cell_id @@ -77,3 +79,35 @@ def _create_bcl_convert_sequencing_metrics( sample_base_mean_quality_score=mean_quality_score, created_at=datetime.now(), ) + + @staticmethod + def create_sample_run_metrics( + sample_internal_id: str, + lane: int, + run_metrics_id: int, + metrics_parser: MetricsParser, + store: Store, + ) -> IlluminaSampleRunMetrics: + """Create sequencing metrics for all lanes in a flow cell.""" + + total_reads: int = metrics_parser.calculate_total_reads_for_sample_in_lane( + sample_internal_id=sample_internal_id, lane=lane + ) + q30_bases_percent: float = metrics_parser.get_q30_bases_percent_for_sample_in_lane( + sample_internal_id=sample_internal_id, lane=lane + ) + mean_quality_score: float = metrics_parser.get_mean_quality_score_for_sample_in_lane( + sample_internal_id=sample_internal_id, lane=lane + ) + sample_id: int = store.get_sample_by_internal_id(sample_internal_id).id + + return IlluminaSampleRunMetrics( + run_metrics_id=run_metrics_id, + sample_id=sample_id, + type=DeviceType.ILLUMINA, + flow_cell_lane=lane, + total_reads_in_lane=total_reads, + base_percentage_passing_q30=q30_bases_percent, + base_mean_quality_score=mean_quality_score, + created_at=datetime.now(), + ) diff --git a/cg/services/illumina_post_processing_service/__init__.py b/cg/services/illumina_post_processing_service/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cg/services/illumina_post_processing_service/illumina_post_processing_service.py b/cg/services/illumina_post_processing_service/illumina_post_processing_service.py new file mode 100644 index 0000000000..3b02987437 --- /dev/null +++ b/cg/services/illumina_post_processing_service/illumina_post_processing_service.py @@ -0,0 +1,103 @@ +"""Module that holds the illumina post-processing service.""" + +import logging +from pathlib import Path + +from cg.apps.housekeeper.hk import HousekeeperAPI +from cg.constants.devices import DeviceType +from cg.exc import MissingFilesError, FlowCellError +from cg.models.flow_cell.flow_cell import FlowCellDirectoryData +from cg.services.illumina_post_processing_service.database_utils import store_illumina_flow_cell +from cg.services.illumina_post_processing_service.utils import ( + create_delivery_file_in_flow_cell_directory, + get_flow_cell_model_from_run_parameters, +) +from cg.services.illumina_post_processing_service.validation import ( + is_flow_cell_ready_for_postprocessing, +) +from cg.store.models import IlluminaFlowCell +from cg.store.store import Store + +LOG = logging.getLogger(__name__) + + +class IlluminaPostProcessingService: + def __init__(self, status_db: Store, housekeeper_api: HousekeeperAPI, dry_run: bool) -> None: + self.status_db: Store = status_db + self.hk_api: HousekeeperAPI = housekeeper_api + self.dry_run: bool = False + + @staticmethod + def store_illumina_flow_cell( + flow_cell: FlowCellDirectoryData, + store: Store, + ) -> IlluminaFlowCell: + """ + Create flow cell from the parsed and validated flow cell data. + And add the samples on the flow cell to the model. + """ + model: str | None = get_flow_cell_model_from_run_parameters(flow_cell.run_parameters_path) + new_flow_cell = IlluminaFlowCell( + internal_id=flow_cell.id, type=DeviceType.ILLUMINA, model=model + ) + return store.add_illumina_flow_cell(new_flow_cell) + + @staticmethod + def store_illumina_sequencing_metrics(flow_cell: IlluminaFlowCell) -> None: + """Store illumina run metrics in the status database.""" + pass + + @staticmethod + def store_illumina_sample_sequencing_metrics(): + """Store illumina sample sequencing metrics in the status database.""" + pass + + def store_illumina_flow_cell_data(self, flow_cell: FlowCellDirectoryData) -> None: + """Store flow cell data in the status database.""" + flow_cell: IlluminaFlowCell = self.store_illumina_flow_cell( + flow_cell=flow_cell, store=self.status_db + ) + self.store_illumina_sequencing_metrics(flow_cell) + self.store_illumina_sample_sequencing_metrics() + self.status_db.commit_to_store() + + def post_process_illumina_flow_cell( + self, + flow_cell_directory_name: str, + demultiplexed_runs_dir: Path, + ) -> None: + """Store data for the demultiplexed flow cell and mark it as ready for delivery. + This function: + - Stores the flow cell in the status database + - Stores sequencing metrics in the status database + - Updates sample read counts in the status database + - Stores the flow cell data in the housekeeper database + - Creates a delivery file in the flow cell directory + Raises: + FlowCellError: If the flow cell directory or the data it contains is not valid. + """ + + LOG.info(f"Post-process flow cell {flow_cell_directory_name}") + flow_cell_out_directory = Path(demultiplexed_runs_dir, flow_cell_directory_name) + flow_cell = FlowCellDirectoryData(flow_cell_out_directory) + sample_sheet_path: Path = self.hk_api.get_sample_sheet_path(flow_cell.id) + flow_cell.set_sample_sheet_path_hk(hk_path=sample_sheet_path) + + LOG.debug("Set path for Housekeeper sample sheet in flow cell") + try: + is_flow_cell_ready_for_postprocessing( + flow_cell_output_directory=flow_cell_out_directory, + flow_cell=flow_cell, + ) + except (FlowCellError, MissingFilesError) as e: + LOG.warning(f"Flow cell {flow_cell_directory_name} will be skipped: {e}") + return + if self.dry_run: + LOG.info(f"Dry run will not finish flow cell {flow_cell_directory_name}") + return + try: + self.store_illumina_flow_cell_data(flow_cell) + except Exception as e: + LOG.error(f"Failed to store flow cell data: {str(e)}") + raise + create_delivery_file_in_flow_cell_directory(flow_cell_out_directory) diff --git a/cg/services/illumina_post_processing_service/utils.py b/cg/services/illumina_post_processing_service/utils.py new file mode 100644 index 0000000000..2b2874dd7a --- /dev/null +++ b/cg/services/illumina_post_processing_service/utils.py @@ -0,0 +1,25 @@ +"""Utility functions for the Illumina post-processing service.""" + +from pathlib import Path +from xml.etree.ElementTree import ElementTree, Element + +from cg.constants.demultiplexing import DemultiplexingDirsAndFiles, RunParametersXMLNodes +from cg.exc import XMLError +from cg.io.xml import read_xml, get_tree_node + + +def create_delivery_file_in_flow_cell_directory(flow_cell_directory: Path) -> None: + Path(flow_cell_directory, DemultiplexingDirsAndFiles.DELIVERY).touch() + + +def get_flow_cell_model_from_run_parameters(run_parameters_path: Path) -> str | None: + """Return the model of the flow cell.""" + xml_tree: ElementTree = read_xml(run_parameters_path) + node: Element | None = None + for node_name in [RunParametersXMLNodes.MODE, RunParametersXMLNodes.FLOW_CELL_MODE]: + try: + node: Element = get_tree_node(xml_tree, node_name) + return node.text + except XMLError: + continue + return node diff --git a/cg/meta/demultiplex/validation.py b/cg/services/illumina_post_processing_service/validation.py similarity index 100% rename from cg/meta/demultiplex/validation.py rename to cg/services/illumina_post_processing_service/validation.py diff --git a/cg/store/crud/create.py b/cg/store/crud/create.py index 7f21ac84f7..addc51a051 100644 --- a/cg/store/crud/create.py +++ b/cg/store/crud/create.py @@ -30,6 +30,7 @@ SampleLaneSequencingMetrics, User, order_case, + IlluminaFlowCell, ) LOG = logging.getLogger(__name__) @@ -415,3 +416,12 @@ def link_case_to_order(order_id: int, case_id: int): session = get_session() session.execute(insert_statement) session.commit() + + def add_illumina_flow_cell(self, flow_cell: IlluminaFlowCell) -> IlluminaFlowCell: + """Add a new Illumina flow cell to the status database as a pending transaction.""" + if self.get_illumina_flow_cell_by_internal_id(flow_cell.internal_id): + raise ValueError(f"Flow cell with {flow_cell.id} already exists.") + session = get_session() + session.add(flow_cell) + LOG.debug(f"Flow cell added to status db: {flow_cell.id}.") + return flow_cell diff --git a/cg/store/crud/read.py b/cg/store/crud/read.py index baf5040455..e51619f6cb 100644 --- a/cg/store/crud/read.py +++ b/cg/store/crud/read.py @@ -50,6 +50,10 @@ FlowCellFilter, apply_flow_cell_filter, ) +from cg.store.filters.status_illumina_flow_cell_filters import ( + apply_illumina_flow_cell_filters, + IlluminaFlowCellFilter, +) from cg.store.filters.status_invoice_filters import InvoiceFilter, apply_invoice_filter from cg.store.filters.status_metrics_filters import ( SequencingMetricsFilter, @@ -84,6 +88,7 @@ Sample, SampleLaneSequencingMetrics, User, + IlluminaFlowCell, ) LOG = logging.getLogger(__name__) @@ -1468,3 +1473,11 @@ def get_case_in_sequencing_count(self, order_id: int) -> int: filter_functions=filters, order_id=order_id, ).count() + + def get_illumina_flow_cell_by_internal_id(self, internal_id: str) -> IlluminaFlowCell: + """Return a flow cell by internal id.""" + return apply_illumina_flow_cell_filters( + filter_functions=[IlluminaFlowCellFilter.BY_INTERNAL_ID], + flow_cells=self._get_query(table=IlluminaFlowCell), + internal_id=internal_id, + ).first() diff --git a/cg/store/filters/status_illumina_flow_cell_filters.py b/cg/store/filters/status_illumina_flow_cell_filters.py new file mode 100644 index 0000000000..d0b77bc77b --- /dev/null +++ b/cg/store/filters/status_illumina_flow_cell_filters.py @@ -0,0 +1,32 @@ +"""Fillters for the Illumina Flow Cells.""" + +from enum import Enum + +from sqlalchemy.orm import Query + +from cg.store.models import IlluminaFlowCell + + +def filter_illumina_flow_cell_by_internal_id( + flow_cells: Query, internal_id: str, **kwargs +) -> Query: + """Get an Illumina Flow Cell by its internal id.""" + return flow_cells.filter_by(internal_id=internal_id) + + +def apply_illumina_flow_cell_filters( + flow_cells: Query, filter_functions: list[callable], internal_id: str +) -> Query: + """Apply filtering functions and return filtered results.""" + for function in filter_functions: + flow_cells: Query = function( + flow_cells=flow_cells, + internal_id=internal_id, + ) + return flow_cells + + +class IlluminaFlowCellFilter(Enum): + """Define FlowCell filter functions.""" + + BY_INTERNAL_ID: callable = filter_illumina_flow_cell_by_internal_id diff --git a/cg/store/store.py b/cg/store/store.py index c99999630a..2b8ff0a00a 100644 --- a/cg/store/store.py +++ b/cg/store/store.py @@ -22,3 +22,7 @@ def __init__(self): DeleteDataHandler(self.session) ReadHandler(self.session) UpdateHandler(self.session) + + def commit_to_store(self): + """Commit pending changes to the store.""" + self.session.commit() diff --git a/tests/meta/demultiplex/test_validation.py b/tests/meta/demultiplex/test_validation.py index 7a3010aaca..8ded5016cf 100644 --- a/tests/meta/demultiplex/test_validation.py +++ b/tests/meta/demultiplex/test_validation.py @@ -5,7 +5,7 @@ from cg.constants import FileExtensions from cg.constants.demultiplexing import DemultiplexingDirsAndFiles from cg.exc import FlowCellError, MissingFilesError -from cg.meta.demultiplex.validation import ( +from cg.services.illumina_post_processing_service.validation import ( is_demultiplexing_complete, is_flow_cell_ready_for_delivery, validate_demultiplexing_complete, diff --git a/tests/services/illumina_post_processing_service/conftest.py b/tests/services/illumina_post_processing_service/conftest.py new file mode 100644 index 0000000000..f530692a81 --- /dev/null +++ b/tests/services/illumina_post_processing_service/conftest.py @@ -0,0 +1,31 @@ +"""Fixtures for the tests of the IlluminaPostProcessingService.""" + +from pathlib import Path + +import pytest + + +@pytest.fixture +def run_paramters_with_flow_cell_mode_node_name() -> Path: + return Path( + "tests", + "fixtures", + "apps", + "demultiplexing", + "flow_cells", + "230912_A00187_1009_AHK33MDRX3", + "RunParameters.xml", + ) + + +@pytest.fixture +def run_parameters_without_model() -> Path: + return Path( + "tests", + "fixtures", + "apps", + "demultiplexing", + "flow_cells", + "170517_ST-E00266_0210_BHJCFFALXX", + "runParameters.xml", + ) diff --git a/tests/services/illumina_post_processing_service/test_illumina_post_processing_utils.py b/tests/services/illumina_post_processing_service/test_illumina_post_processing_utils.py new file mode 100644 index 0000000000..b4e6ed7cc7 --- /dev/null +++ b/tests/services/illumina_post_processing_service/test_illumina_post_processing_utils.py @@ -0,0 +1,43 @@ +"""Test utils of the illumina post processing service.""" + +from pathlib import Path + +from cg.services.illumina_post_processing_service.utils import ( + get_flow_cell_model_from_run_parameters, +) + + +def test_get_flow_cell_model_from_run_parameters_with_mode_node_name( + novaseq_x_run_parameters_path: Path, +): + # GIVEN a run parameters file from a NovaSeq X run + + # WHEN getting the flow cell model from the run parameters file + flow_cell_model = get_flow_cell_model_from_run_parameters(novaseq_x_run_parameters_path) + + # THEN the flow cell model should be returned + assert flow_cell_model == "10B" + + +def test_get_flow_cel_model_from_run_parameters_with_flow_cell_mode_node_name( + run_paramters_with_flow_cell_mode_node_name: Path, +): + # GIVEN a run parameters file with a FlowCellMode node + + # WHEN getting the flow cell model from the run parameters file + flow_cell_model = get_flow_cell_model_from_run_parameters( + run_paramters_with_flow_cell_mode_node_name + ) + + # THEN the flow cell model should be returned + assert flow_cell_model == "S1" + + +def test_get_flow_cell_model_from_run_paramters_without_model(run_parameters_without_model: Path): + # GIVEN a run parameters file without a model + + # WHEN getting the flow cell model from the run parameters file + flow_cell_model = get_flow_cell_model_from_run_parameters(run_parameters_without_model) + + # THEN the flow cell model should be None + assert flow_cell_model is None diff --git a/tests/store/conftest.py b/tests/store/conftest.py index 752af58ddd..80f7b508e5 100644 --- a/tests/store/conftest.py +++ b/tests/store/conftest.py @@ -8,6 +8,7 @@ import pytest from cg.constants import Workflow +from cg.constants.devices import DeviceType from cg.constants.priority import PriorityTerms from cg.constants.subject import PhenotypeStatus, Sex from cg.meta.orders.pool_submitter import PoolSubmitter @@ -19,6 +20,7 @@ Customer, Organism, Sample, + IlluminaFlowCell, ) from cg.store.store import Store from tests.store_helpers import StoreHelpers @@ -586,3 +588,24 @@ def store_with_samples_for_multiple_customers( delivered_at=timestamp_now, ) yield store + + +@pytest.fixture +def illumina_flow_cell_internal_id() -> str: + return "FC123456" + + +@pytest.fixture +def illumina_flow_cell_model_s1() -> str: + return "S1" + + +@pytest.fixture +def illumina_flow_cell( + illumina_flow_cell_internal_id: str, illumina_flow_cell_model_s1: str +) -> IlluminaFlowCell: + return IlluminaFlowCell( + internal_id=illumina_flow_cell_internal_id, + type=DeviceType.ILLUMINA, + model=illumina_flow_cell_model_s1, + ) diff --git a/tests/store/crud/add/test_store_add_base.py b/tests/store/crud/add/test_store_add_base.py index f619d6b4a0..bb0063b72b 100644 --- a/tests/store/crud/add/test_store_add_base.py +++ b/tests/store/crud/add/test_store_add_base.py @@ -1,5 +1,7 @@ from datetime import datetime as dt +import pytest + from cg.constants.subject import Sex from cg.store.models import ( ApplicationVersion, @@ -8,6 +10,7 @@ Organism, Sample, User, + IlluminaFlowCell, ) from cg.store.store import Store @@ -114,3 +117,32 @@ def test_add_pool(rml_pool_store: Store): # THEN the new pool should have no_invoice = False pool = rml_pool_store.get_pool_by_entry_id(entry_id=2) assert pool.no_invoice is False + + +def test_add_illumina_flow_cell( + illumina_flow_cell: IlluminaFlowCell, illumina_flow_cell_internal_id: str, store: Store +): + # GIVEN an Illumina flow cell not in store + assert not store.get_illumina_flow_cell_by_internal_id(illumina_flow_cell_internal_id) + + # WHEN adding the flow cell to the store + store.add_illumina_flow_cell(illumina_flow_cell) + + # THEN it should be stored in the database + assert ( + store.get_illumina_flow_cell_by_internal_id(illumina_flow_cell_internal_id) + == illumina_flow_cell + ) + + +def test_add_illumina_flow_cell_already_exists( + illumina_flow_cell: IlluminaFlowCell, illumina_flow_cell_internal_id: str, store: Store +): + # GIVEN an Illumina flow cell not in store + store.add_illumina_flow_cell(illumina_flow_cell) + assert store.get_illumina_flow_cell_by_internal_id(illumina_flow_cell_internal_id) + + # WHEN adding the flow cell to the store + # THEN a ValueError should be raised + with pytest.raises(ValueError): + store.add_illumina_flow_cell(illumina_flow_cell) diff --git a/tests/store/crud/conftest.py b/tests/store/crud/conftest.py index 9cd9510914..e1458c1e26 100644 --- a/tests/store/crud/conftest.py +++ b/tests/store/crud/conftest.py @@ -1,8 +1,6 @@ from datetime import datetime, timedelta from typing import Generator - import pytest - from cg.constants import Workflow from cg.constants.constants import CustomerId, PrepCategory from cg.constants.subject import PhenotypeStatus diff --git a/tests/store/crud/read/test_read_illumina_flow_cell.py b/tests/store/crud/read/test_read_illumina_flow_cell.py new file mode 100644 index 0000000000..118efe9848 --- /dev/null +++ b/tests/store/crud/read/test_read_illumina_flow_cell.py @@ -0,0 +1,20 @@ +"""Tests for the read illumina flow cell functions in the store.""" + +from cg.store.models import IlluminaFlowCell +from cg.store.store import Store + + +def test_get_illumina_flow_cell_by_internal_id( + illumina_flow_cell: IlluminaFlowCell, illumina_flow_cell_internal_id: str, store: Store +): + # GIVEN an Illumina flow cell in store + store.add_illumina_flow_cell(illumina_flow_cell) + + # WHEN getting the flow cell from the store + flow_cell: IlluminaFlowCell = store.get_illumina_flow_cell_by_internal_id( + illumina_flow_cell_internal_id + ) + + # THEN the correct flow cell should be returned + assert isinstance(flow_cell, IlluminaFlowCell) + assert flow_cell.internal_id == illumina_flow_cell_internal_id diff --git a/tests/store/filters/test_status_illumina_flow_cell_filters.py b/tests/store/filters/test_status_illumina_flow_cell_filters.py new file mode 100644 index 0000000000..384605708b --- /dev/null +++ b/tests/store/filters/test_status_illumina_flow_cell_filters.py @@ -0,0 +1,27 @@ +"""Tests for the illumina flow cell filters in the store.""" + +from sqlalchemy.orm import Query + +from cg.store.filters.status_illumina_flow_cell_filters import ( + filter_illumina_flow_cell_by_internal_id, +) +from cg.store.models import IlluminaFlowCell +from cg.store.store import Store + + +def test_filter_illumina_flow_cell_by_internal_id( + illumina_flow_cell: IlluminaFlowCell, illumina_flow_cell_internal_id: str, store: Store +): + # GIVEN an Illumina flow cell in store + store.add_illumina_flow_cell(illumina_flow_cell) + + # WHEN filtering an Illumina flow cell by internal id + flow_cells: Query = store._get_query(table=IlluminaFlowCell) + flow_cell: Query = filter_illumina_flow_cell_by_internal_id( + flow_cells=flow_cells, internal_id=illumina_flow_cell_internal_id + ) + + # THEN a query with the correct flow cell should be returned + assert isinstance(flow_cell, Query) + assert flow_cell.first() == illumina_flow_cell + assert flow_cell.first().internal_id == illumina_flow_cell_internal_id From f07d1e72496b35cb14eb519ad5841b6ee915ab52 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Wed, 22 May 2024 07:44:43 +0000 Subject: [PATCH 32/42] =?UTF-8?q?Bump=20version:=2060.8.1=20=E2=86=92=2060?= =?UTF-8?q?.8.2=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c17778e6ed..fa43410ff9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.8.1 +current_version = 60.8.2 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index d858f49281..c12ee7dd2b 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.8.1" +__version__ = "60.8.2" diff --git a/pyproject.toml b/pyproject.toml index 57d73fce14..7100f59e8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.8.1" +version = "60.8.2" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From eb23c886270e81726c103f31b03a69d4ce146581 Mon Sep 17 00:00:00 2001 From: EliottBo <112384714+eliottBo@users.noreply.github.com> Date: Wed, 22 May 2024 10:12:34 +0200 Subject: [PATCH 33/42] Remove dryrun flag (#3244) (patch) ###Changed - Removes the short flag "-d" in the DRY_RUN option in cg/constants/constants.py - In test files, "-d" was changed to "--dry-run". --- cg/constants/constants.py | 1 - tests/cli/clean/test_balsamic_clean.py | 6 +++-- tests/cli/clean/test_clean_flow_cell.py | 2 +- tests/cli/clean/test_microbial_clean.py | 2 +- tests/cli/clean/test_rsync_past_run_dirs.py | 2 +- .../microsalt/test_microsalt_case_config.py | 26 ++++++++++++++----- .../workflow/microsalt/test_microsalt_run.py | 2 +- .../nf_analysis/test_cli_config_case.py | 4 ++- 8 files changed, 30 insertions(+), 15 deletions(-) diff --git a/cg/constants/constants.py b/cg/constants/constants.py index da9801c946..d3df3ed866 100644 --- a/cg/constants/constants.py +++ b/cg/constants/constants.py @@ -234,7 +234,6 @@ class APIMethods(StrEnum): DRY_RUN = click.option( - "-d", "--dry-run", is_flag=True, default=False, diff --git a/tests/cli/clean/test_balsamic_clean.py b/tests/cli/clean/test_balsamic_clean.py index 20c11f6bb4..8443f39a61 100644 --- a/tests/cli/clean/test_balsamic_clean.py +++ b/tests/cli/clean/test_balsamic_clean.py @@ -96,7 +96,9 @@ def test_dry_run( TrailblazerAPI.is_latest_analysis_ongoing.return_value = False # WHEN dry running with dry run specified - result = cli_runner.invoke(clean_run_dir, [balsamic_case_clean, "-d", "-y"], obj=clean_context) + result = cli_runner.invoke( + clean_run_dir, [balsamic_case_clean, "--dry-run", "-y"], obj=clean_context + ) # THEN command should say it would have deleted assert result.exit_code == EXIT_SUCCESS assert "Would have deleted" in caplog.text @@ -136,7 +138,7 @@ def test_cleaned_at_invalid( assert not base_store.get_case_by_internal_id(balsamic_case_not_clean).analyses[0].cleaned_at # WHEN dry running with dry run specified - result = cli_runner.invoke(past_run_dirs, ["2020-12-01", "-d", "-y"], obj=clean_context) + result = cli_runner.invoke(past_run_dirs, ["2020-12-01", "--dry-run", "-y"], obj=clean_context) # THEN case directory should not have been cleaned assert result.exit_code == EXIT_SUCCESS diff --git a/tests/cli/clean/test_clean_flow_cell.py b/tests/cli/clean/test_clean_flow_cell.py index 1e65e73122..e7d4e86450 100644 --- a/tests/cli/clean/test_clean_flow_cell.py +++ b/tests/cli/clean/test_clean_flow_cell.py @@ -62,7 +62,7 @@ def test_clean_flow_cells_cmd_dry_run( "cg.meta.clean.clean_flow_cells.CleanFlowCellAPI.is_directory_older_than_21_days", return_value=True, ): - result = cli_runner.invoke(clean_flow_cells, ["-d"], obj=clean_flow_cells_context) + result = cli_runner.invoke(clean_flow_cells, ["--dry-run"], obj=clean_flow_cells_context) # THEN assert it exits with success assert result.exit_code == 0 diff --git a/tests/cli/clean/test_microbial_clean.py b/tests/cli/clean/test_microbial_clean.py index 1fe668d444..fa0229d7a6 100644 --- a/tests/cli/clean/test_microbial_clean.py +++ b/tests/cli/clean/test_microbial_clean.py @@ -48,7 +48,7 @@ def test_dry_run( # WHEN dry running with dry run specified result = cli_runner.invoke( - clean_run_dir, [microsalt_case_clean_dry, "-d", "-y"], obj=clean_context_microsalt + clean_run_dir, [microsalt_case_clean_dry, "--dry-run", "-y"], obj=clean_context_microsalt ) # THEN command should say it would have deleted diff --git a/tests/cli/clean/test_rsync_past_run_dirs.py b/tests/cli/clean/test_rsync_past_run_dirs.py index e1239bb3ef..a6d7801007 100644 --- a/tests/cli/clean/test_rsync_past_run_dirs.py +++ b/tests/cli/clean/test_rsync_past_run_dirs.py @@ -26,7 +26,7 @@ def test_clean_rsync_past_run_dirs_young_process( assert rsync_process.exists() # WHEN attempting to remove said process same day result = cli_runner.invoke( - rsync_past_run_dirs, ["-d", "-y", str(timestamp_now)], obj=clean_context + rsync_past_run_dirs, ["--dry-run", "-y", str(timestamp_now)], obj=clean_context ) # THEN the command should not fail and notice that the process is not old assert result.exit_code == EXIT_SUCCESS diff --git a/tests/cli/workflow/microsalt/test_microsalt_case_config.py b/tests/cli/workflow/microsalt/test_microsalt_case_config.py index ca20bcd1a9..88da8b019d 100644 --- a/tests/cli/workflow/microsalt/test_microsalt_case_config.py +++ b/tests/cli/workflow/microsalt/test_microsalt_case_config.py @@ -87,7 +87,9 @@ def test_dry_sample( # GIVEN project, organism and reference genome is specified in lims # WHEN dry running a sample name - result = cli_runner.invoke(config_case, [microbial_sample_id, "-s", "-d"], obj=base_context) + result = cli_runner.invoke( + config_case, [microbial_sample_id, "-s", "--dry-run"], obj=base_context + ) # THEN command should give us a json dump assert result.exit_code == EXIT_SUCCESS @@ -105,7 +107,7 @@ def test_dry_order( # WHEN dry running a sample name result = cli_runner.invoke( config_case, - [ticket_id, "-t", "-d"], + [ticket_id, "-t", "--dry-run"], obj=base_context, ) @@ -135,7 +137,9 @@ def test_gonorrhoeae(cli_runner: CliRunner, base_context: CGConfig, microbial_sa sample_obj.organism.internal_id = "gonorrhoeae" # WHEN getting the case config - result = cli_runner.invoke(config_case, [microbial_sample_id, "-d", "-s"], obj=base_context) + result = cli_runner.invoke( + config_case, [microbial_sample_id, "--dry-run", "-s"], obj=base_context + ) # THEN the organism should now be ... assert "Neisseria spp." in result.output @@ -150,7 +154,9 @@ def test_cutibacterium_acnes(cli_runner: CliRunner, base_context: CGConfig, micr sample_obj.organism.internal_id = "Cutibacterium acnes" # WHEN getting the case config - result = cli_runner.invoke(config_case, [microbial_sample_id, "-s", "-d"], obj=base_context) + result = cli_runner.invoke( + config_case, [microbial_sample_id, "-s", "--dry-run"], obj=base_context + ) # THEN the organism should now be .... assert "Propionibacterium acnes" in result.output @@ -166,7 +172,9 @@ def test_vre_nc_017960(cli_runner: CliRunner, base_context: CGConfig, microbial_ sample_obj.organism.reference_genome = "NC_017960.1" # WHEN getting the case config - result = cli_runner.invoke(config_case, [microbial_sample_id, "-s", "-d"], obj=base_context) + result = cli_runner.invoke( + config_case, [microbial_sample_id, "-s", "--dry-run"], obj=base_context + ) # THEN the organism should now be .... assert "Enterococcus faecium" in result.output @@ -182,7 +190,9 @@ def test_vre_nc_004668(cli_runner: CliRunner, base_context: CGConfig, microbial_ sample_obj.organism.reference_genome = "NC_004668.1" # WHEN getting the case config - result = cli_runner.invoke(config_case, [microbial_sample_id, "-s", "-d"], obj=base_context) + result = cli_runner.invoke( + config_case, [microbial_sample_id, "-s", "--dry-run"], obj=base_context + ) # THEN the organism should now be .... assert "Enterococcus faecalis" in result.output @@ -201,7 +211,9 @@ def test_vre_comment( lims_sample.sample_data["comment"] = "ABCD123" # WHEN getting the case config - result = cli_runner.invoke(config_case, [microbial_sample_id, "-s", "-d"], obj=base_context) + result = cli_runner.invoke( + config_case, [microbial_sample_id, "-s", "--dry-run"], obj=base_context + ) # THEN the organism should now be .... assert "ABCD123" in result.output diff --git a/tests/cli/workflow/microsalt/test_microsalt_run.py b/tests/cli/workflow/microsalt/test_microsalt_run.py index cc02eb88ac..fe1d590e86 100644 --- a/tests/cli/workflow/microsalt/test_microsalt_run.py +++ b/tests/cli/workflow/microsalt/test_microsalt_run.py @@ -29,7 +29,7 @@ def test_dry_arguments(cli_runner: CliRunner, base_context: CGConfig, ticket_id, caplog.set_level(logging.INFO) # WHEN dry running without anything specified - result = cli_runner.invoke(run, [ticket_id, "-t", "-d"], obj=base_context) + result = cli_runner.invoke(run, [ticket_id, "-t", "--dry-run"], obj=base_context) # THEN command should mention missing arguments assert result.exit_code == EXIT_SUCCESS diff --git a/tests/cli/workflow/nf_analysis/test_cli_config_case.py b/tests/cli/workflow/nf_analysis/test_cli_config_case.py index 4d39c232eb..2cad28dff0 100644 --- a/tests/cli/workflow/nf_analysis/test_cli_config_case.py +++ b/tests/cli/workflow/nf_analysis/test_cli_config_case.py @@ -212,7 +212,9 @@ def test_config_case_dry_run( mocker.patch.object(LimsAPI, "get_source", return_value="blood") # WHEN invoking the command with dry-run specified - result = cli_runner.invoke(workflow_cli, [workflow, "config-case", case_id, "-d"], obj=context) + result = cli_runner.invoke( + workflow_cli, [workflow, "config-case", case_id, "--dry-run"], obj=context + ) # THEN command should exit successfully assert result.exit_code == EXIT_SUCCESS From f7dc8b05417802de11119798d5ed55ef52a0e632 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Wed, 22 May 2024 08:13:02 +0000 Subject: [PATCH 34/42] =?UTF-8?q?Bump=20version:=2060.8.2=20=E2=86=92=2060?= =?UTF-8?q?.8.3=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fa43410ff9..8038f35067 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.8.2 +current_version = 60.8.3 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index c12ee7dd2b..56148e35e9 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.8.2" +__version__ = "60.8.3" diff --git a/pyproject.toml b/pyproject.toml index 7100f59e8f..a5a455dfa1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.8.2" +version = "60.8.3" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 440a93b0b3896abc413760e333d9069a0b3f26f2 Mon Sep 17 00:00:00 2001 From: ChristianOertlin Date: Thu, 23 May 2024 10:21:01 +0200 Subject: [PATCH 35/42] prepend date alembic scripts (#3266) (patch) renaming alembic files --- ..._02_02_6d74453565f2_add_retrieved_status_to_flowcell_table.py} | 0 ...o_case.py => 2021_02_17_fab30255b84f_move_synopsis_to_case.py} | 0 ...s.py => 2021_02_26_998be2e367cf_fix_mip_on_fastq_wgs_cases.py} | 0 ...3_10_49ded71bd1a1_add_uploaded_to_vogue_column_to_analysis.py} | 0 ...3_16_432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py} | 0 ...user.py => 2021_03_23_089edc289291_add_table_customer_user.py} | 0 ...to_maf.py => 2021_03_23_e9df15a35de4_fix_tumour_not_to_maf.py} | 0 ...etname_avatar.py => 2021_04_08_7e344b9438bf_petname_avatar.py} | 0 ...ails.py => 2021_04_19_1dadcefd3bbf_feat_contacts_as_emails.py} | 0 ...dd_user_types.py => 2021_05_03_ed0be7286cee_add_user_types.py} | 0 ...py => 2021_05_03_ef472ce31931_add_name_for_invoice_contact.py} | 0 ...1c27462b49f6_add_phenotype_groups_and_subject_id_to_sample.py} | 0 ...sample.py => 2021_09_10_c081458ff180_add_control_to_sample.py} | 0 ...ns.py => 2021_09_28_c494649637d5_add_data_delivery_options.py} | 0 ...num.py => 2021_12_16_f2edbd530656_convert_priority_to_enum.py} | 0 ...2_01_28_c76d655c8edf_add_fastq_qc_analysis_option_to_data_.py} | 0 ...na.py => 2022_02_10_76e8252a6efb_add_data_delivery_statina.py} | 0 ...emove_avatars.py => 2022_02_25_bf97b0121538_remove_avatars.py} | 0 ...nical_trials.py => 2022_03_28_0baf8309d227_clinical_trials.py} | 0 ...f_1508_26.py => 2022_04_06_367813f2e597_support_of_1508_26.py} | 0 ...ion.py => 2022_06_28_33cd4b45acb4_add_data_analysis_option.py} | 0 ...2_06_29_ddc94088be4d_adds_customer_to_customer_group_link_.py} | 0 ....py => 2022_07_19_2968d39ac35f_changes_in_delivery_options.py} | 0 ...ilies.py => 2022_07_22_20750539a335_add_ticket_to_families.py} | 0 ...py => 2022_07_22_9c9ca9407227_add_data_analysis_pon_option.py} | 0 ...y => 2022_09_27_554140bc13e4_add_rnafusion_analysis_option.py} | 0 ...d.py => 2023_04_12_df1b3dd317d0_add_trusted_customer_field.py} | 0 ...=> 2023_04_19_9008aa5065b4_add_taxprofiler_analysis_option.py} | 0 ...ncing_stats.py => 2023_05_04_ea5470295689_sequencing_stats.py} | 0 ....py => 2023_05_26_f5e0db62a5a7_update_sequencingstatistics.py} | 0 ....py => 2023_06_01_e6a3f1ad4b50_change_read_count_to_bigint.py} | 0 ....py => 2023_06_02_367ed257e4ee_change_data_type_of_metrics.py} | 0 ...3_06_02_869d352cc614_add_relationship_between_metrics_and_.py} | 0 ...=> 2023_06_02_a34039f530c5_alter_column_types_sample_lane_.py} | 0 ...y => 2023_06_27_97ffd22d7ebc_add_metrics_unique_constraint.py} | 0 ...p_read_col.py => 2023_06_29_ffb9f8ab8e62_add_temp_read_col.py} | 0 ...on.py => 2023_07_04_c3fdf3a8a5b3_add_data_archive_location.py} | 0 ....py => 2023_07_07_68e54d17f4f3_add_is_clinical_to_customer.py} | 0 ...> 2023_07_10_2fdb42ba801a_support_novaseqx_flow_cell_table.py} | 0 ...umns.py => 2023_08_01_201b16c45366_drop_calc_reads_columns.py} | 0 ...mn.py => 2023_08_18_5cb3ce4c3e39_rename_q30_metrics_column.py} | 0 ...> 2023_09_04_9def7a6eae73_sequenced_at_to_reads_updated_at.py} | 0 ...tatusdb.py => 2023_09_15_936887038ba8_sync_cg_and_statusdb.py} | 0 .../{d0aa961845b9_.py => 2023_09_19_d0aa961845b9_sync_naming.py} | 0 ...p_field.py => 2023_09_22_c3da223e60d8_add_has_backup_field.py} | 0 ... => 2023_09_28_e853d21feaa0_add_pipeline_limitations_table.py} | 0 ...act.py => 2023_10_05_db61c62d9bc0_add_customer_lab_contact.py} | 0 ...> 2023_10_25_392e49db40fc_add_sample_concentration_min_max.py} | 0 ...023_10_31_9073c61bc72b_add_raredisease_to_analysis_options.py} | 0 ...h_tables.py => 2024_05_03_9b188aee9577_add_new_tech_tables.py} | 0 ...2024_05_21_5fd7e8758fb1_add_autoincrement_to_device_tables.py} | 0 51 files changed, 0 insertions(+), 0 deletions(-) rename alembic/versions/{6d74453565f2_add_retrieved_status_to_flowcell_table.py => 2021_02_02_6d74453565f2_add_retrieved_status_to_flowcell_table.py} (100%) rename alembic/versions/{fab30255b84f_move_synopsis_to_case.py => 2021_02_17_fab30255b84f_move_synopsis_to_case.py} (100%) rename alembic/versions/{998be2e367cf_fix_mip_on_fastq_wgs_cases.py => 2021_02_26_998be2e367cf_fix_mip_on_fastq_wgs_cases.py} (100%) rename alembic/versions/{49ded71bd1a1_add_uploaded_to_vogue_column_to_analysis.py => 2021_03_10_49ded71bd1a1_add_uploaded_to_vogue_column_to_analysis.py} (100%) rename alembic/versions/{432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py => 2021_03_16_432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py} (100%) rename alembic/versions/{089edc289291_add_table_customer_user.py => 2021_03_23_089edc289291_add_table_customer_user.py} (100%) rename alembic/versions/{e9df15a35de4_fix_tumour_not_to_maf.py => 2021_03_23_e9df15a35de4_fix_tumour_not_to_maf.py} (100%) rename alembic/versions/{7e344b9438bf_petname_avatar.py => 2021_04_08_7e344b9438bf_petname_avatar.py} (100%) rename alembic/versions/{1dadcefd3bbf_feat_contacts_as_emails.py => 2021_04_19_1dadcefd3bbf_feat_contacts_as_emails.py} (100%) rename alembic/versions/{ed0be7286cee_add_user_types.py => 2021_05_03_ed0be7286cee_add_user_types.py} (100%) rename alembic/versions/{ef472ce31931_add_name_for_invoice_contact.py => 2021_05_03_ef472ce31931_add_name_for_invoice_contact.py} (100%) rename alembic/versions/{1c27462b49f6_add_phenotype_groups_and_subject_id_to_sample.py => 2021_06_30_1c27462b49f6_add_phenotype_groups_and_subject_id_to_sample.py} (100%) rename alembic/versions/{c081458ff180_add_control_to_sample.py => 2021_09_10_c081458ff180_add_control_to_sample.py} (100%) rename alembic/versions/{c494649637d5_add_data_delivery_options.py => 2021_09_28_c494649637d5_add_data_delivery_options.py} (100%) rename alembic/versions/{f2edbd530656_convert_priority_to_enum.py => 2021_12_16_f2edbd530656_convert_priority_to_enum.py} (100%) rename alembic/versions/{c76d655c8edf_add_fastq_qc_analysis_option_to_data_.py => 2022_01_28_c76d655c8edf_add_fastq_qc_analysis_option_to_data_.py} (100%) rename alembic/versions/{76e8252a6efb_add_data_delivery_statina.py => 2022_02_10_76e8252a6efb_add_data_delivery_statina.py} (100%) rename alembic/versions/{bf97b0121538_remove_avatars.py => 2022_02_25_bf97b0121538_remove_avatars.py} (100%) rename alembic/versions/{0baf8309d227_clinical_trials.py => 2022_03_28_0baf8309d227_clinical_trials.py} (100%) rename alembic/versions/{367813f2e597_support_of_1508_26.py => 2022_04_06_367813f2e597_support_of_1508_26.py} (100%) rename alembic/versions/{33cd4b45acb4_add_data_analysis_option.py => 2022_06_28_33cd4b45acb4_add_data_analysis_option.py} (100%) rename alembic/versions/{ddc94088be4d_adds_customer_to_customer_group_link_.py => 2022_06_29_ddc94088be4d_adds_customer_to_customer_group_link_.py} (100%) rename alembic/versions/{2968d39ac35f_changes_in_delivery_options.py => 2022_07_19_2968d39ac35f_changes_in_delivery_options.py} (100%) rename alembic/versions/{20750539a335_add_ticket_to_families.py => 2022_07_22_20750539a335_add_ticket_to_families.py} (100%) rename alembic/versions/{9c9ca9407227_add_data_analysis_pon_option.py => 2022_07_22_9c9ca9407227_add_data_analysis_pon_option.py} (100%) rename alembic/versions/{554140bc13e4_add_rnafusion_analysis_option.py => 2022_09_27_554140bc13e4_add_rnafusion_analysis_option.py} (100%) rename alembic/versions/{df1b3dd317d0_add_trusted_customer_field.py => 2023_04_12_df1b3dd317d0_add_trusted_customer_field.py} (100%) rename alembic/versions/{9008aa5065b4_add_taxprofiler_analysis_option.py => 2023_04_19_9008aa5065b4_add_taxprofiler_analysis_option.py} (100%) rename alembic/versions/{ea5470295689_sequencing_stats.py => 2023_05_04_ea5470295689_sequencing_stats.py} (100%) rename alembic/versions/{f5e0db62a5a7_update_sequencingstatistics.py => 2023_05_26_f5e0db62a5a7_update_sequencingstatistics.py} (100%) rename alembic/versions/{e6a3f1ad4b50_change_read_count_to_bigint.py => 2023_06_01_e6a3f1ad4b50_change_read_count_to_bigint.py} (100%) rename alembic/versions/{367ed257e4ee_change_data_type_of_metrics.py => 2023_06_02_367ed257e4ee_change_data_type_of_metrics.py} (100%) rename alembic/versions/{869d352cc614_add_relationship_between_metrics_and_.py => 2023_06_02_869d352cc614_add_relationship_between_metrics_and_.py} (100%) rename alembic/versions/{a34039f530c5_alter_column_types_sample_lane_.py => 2023_06_02_a34039f530c5_alter_column_types_sample_lane_.py} (100%) rename alembic/versions/{97ffd22d7ebc_add_metrics_unique_constraint.py => 2023_06_27_97ffd22d7ebc_add_metrics_unique_constraint.py} (100%) rename alembic/versions/{ffb9f8ab8e62_add_temp_read_col.py => 2023_06_29_ffb9f8ab8e62_add_temp_read_col.py} (100%) rename alembic/versions/{c3fdf3a8a5b3_add_data_archive_location.py => 2023_07_04_c3fdf3a8a5b3_add_data_archive_location.py} (100%) rename alembic/versions/{68e54d17f4f3_add_is_clinical_to_customer.py => 2023_07_07_68e54d17f4f3_add_is_clinical_to_customer.py} (100%) rename alembic/versions/{2fdb42ba801a_support_novaseqx_flow_cell_table.py => 2023_07_10_2fdb42ba801a_support_novaseqx_flow_cell_table.py} (100%) rename alembic/versions/{201b16c45366_drop_calc_reads_columns.py => 2023_08_01_201b16c45366_drop_calc_reads_columns.py} (100%) rename alembic/versions/{5cb3ce4c3e39_rename_q30_metrics_column.py => 2023_08_18_5cb3ce4c3e39_rename_q30_metrics_column.py} (100%) rename alembic/versions/{9def7a6eae73_sequenced_at_to_reads_updated_at.py => 2023_09_04_9def7a6eae73_sequenced_at_to_reads_updated_at.py} (100%) rename alembic/versions/{936887038ba8_sync_cg_and_statusdb.py => 2023_09_15_936887038ba8_sync_cg_and_statusdb.py} (100%) rename alembic/versions/{d0aa961845b9_.py => 2023_09_19_d0aa961845b9_sync_naming.py} (100%) rename alembic/versions/{c3da223e60d8_add_has_backup_field.py => 2023_09_22_c3da223e60d8_add_has_backup_field.py} (100%) rename alembic/versions/{e853d21feaa0_add_pipeline_limitations_table.py => 2023_09_28_e853d21feaa0_add_pipeline_limitations_table.py} (100%) rename alembic/versions/{db61c62d9bc0_add_customer_lab_contact.py => 2023_10_05_db61c62d9bc0_add_customer_lab_contact.py} (100%) rename alembic/versions/{392e49db40fc_add_sample_concentration_min_max.py => 2023_10_25_392e49db40fc_add_sample_concentration_min_max.py} (100%) rename alembic/versions/{9073c61bc72b_add_raredisease_to_analysis_options.py => 2023_10_31_9073c61bc72b_add_raredisease_to_analysis_options.py} (100%) rename alembic/versions/{9b188aee9577_add_new_tech_tables.py => 2024_05_03_9b188aee9577_add_new_tech_tables.py} (100%) rename alembic/versions/{5fd7e8758fb1_add_autoincrement_to_device_tables.py => 2024_05_21_5fd7e8758fb1_add_autoincrement_to_device_tables.py} (100%) diff --git a/alembic/versions/6d74453565f2_add_retrieved_status_to_flowcell_table.py b/alembic/versions/2021_02_02_6d74453565f2_add_retrieved_status_to_flowcell_table.py similarity index 100% rename from alembic/versions/6d74453565f2_add_retrieved_status_to_flowcell_table.py rename to alembic/versions/2021_02_02_6d74453565f2_add_retrieved_status_to_flowcell_table.py diff --git a/alembic/versions/fab30255b84f_move_synopsis_to_case.py b/alembic/versions/2021_02_17_fab30255b84f_move_synopsis_to_case.py similarity index 100% rename from alembic/versions/fab30255b84f_move_synopsis_to_case.py rename to alembic/versions/2021_02_17_fab30255b84f_move_synopsis_to_case.py diff --git a/alembic/versions/998be2e367cf_fix_mip_on_fastq_wgs_cases.py b/alembic/versions/2021_02_26_998be2e367cf_fix_mip_on_fastq_wgs_cases.py similarity index 100% rename from alembic/versions/998be2e367cf_fix_mip_on_fastq_wgs_cases.py rename to alembic/versions/2021_02_26_998be2e367cf_fix_mip_on_fastq_wgs_cases.py diff --git a/alembic/versions/49ded71bd1a1_add_uploaded_to_vogue_column_to_analysis.py b/alembic/versions/2021_03_10_49ded71bd1a1_add_uploaded_to_vogue_column_to_analysis.py similarity index 100% rename from alembic/versions/49ded71bd1a1_add_uploaded_to_vogue_column_to_analysis.py rename to alembic/versions/2021_03_10_49ded71bd1a1_add_uploaded_to_vogue_column_to_analysis.py diff --git a/alembic/versions/432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py b/alembic/versions/2021_03_16_432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py similarity index 100% rename from alembic/versions/432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py rename to alembic/versions/2021_03_16_432379a1adfa_fix_sars_cov_2_data_analysis_to_database.py diff --git a/alembic/versions/089edc289291_add_table_customer_user.py b/alembic/versions/2021_03_23_089edc289291_add_table_customer_user.py similarity index 100% rename from alembic/versions/089edc289291_add_table_customer_user.py rename to alembic/versions/2021_03_23_089edc289291_add_table_customer_user.py diff --git a/alembic/versions/e9df15a35de4_fix_tumour_not_to_maf.py b/alembic/versions/2021_03_23_e9df15a35de4_fix_tumour_not_to_maf.py similarity index 100% rename from alembic/versions/e9df15a35de4_fix_tumour_not_to_maf.py rename to alembic/versions/2021_03_23_e9df15a35de4_fix_tumour_not_to_maf.py diff --git a/alembic/versions/7e344b9438bf_petname_avatar.py b/alembic/versions/2021_04_08_7e344b9438bf_petname_avatar.py similarity index 100% rename from alembic/versions/7e344b9438bf_petname_avatar.py rename to alembic/versions/2021_04_08_7e344b9438bf_petname_avatar.py diff --git a/alembic/versions/1dadcefd3bbf_feat_contacts_as_emails.py b/alembic/versions/2021_04_19_1dadcefd3bbf_feat_contacts_as_emails.py similarity index 100% rename from alembic/versions/1dadcefd3bbf_feat_contacts_as_emails.py rename to alembic/versions/2021_04_19_1dadcefd3bbf_feat_contacts_as_emails.py diff --git a/alembic/versions/ed0be7286cee_add_user_types.py b/alembic/versions/2021_05_03_ed0be7286cee_add_user_types.py similarity index 100% rename from alembic/versions/ed0be7286cee_add_user_types.py rename to alembic/versions/2021_05_03_ed0be7286cee_add_user_types.py diff --git a/alembic/versions/ef472ce31931_add_name_for_invoice_contact.py b/alembic/versions/2021_05_03_ef472ce31931_add_name_for_invoice_contact.py similarity index 100% rename from alembic/versions/ef472ce31931_add_name_for_invoice_contact.py rename to alembic/versions/2021_05_03_ef472ce31931_add_name_for_invoice_contact.py diff --git a/alembic/versions/1c27462b49f6_add_phenotype_groups_and_subject_id_to_sample.py b/alembic/versions/2021_06_30_1c27462b49f6_add_phenotype_groups_and_subject_id_to_sample.py similarity index 100% rename from alembic/versions/1c27462b49f6_add_phenotype_groups_and_subject_id_to_sample.py rename to alembic/versions/2021_06_30_1c27462b49f6_add_phenotype_groups_and_subject_id_to_sample.py diff --git a/alembic/versions/c081458ff180_add_control_to_sample.py b/alembic/versions/2021_09_10_c081458ff180_add_control_to_sample.py similarity index 100% rename from alembic/versions/c081458ff180_add_control_to_sample.py rename to alembic/versions/2021_09_10_c081458ff180_add_control_to_sample.py diff --git a/alembic/versions/c494649637d5_add_data_delivery_options.py b/alembic/versions/2021_09_28_c494649637d5_add_data_delivery_options.py similarity index 100% rename from alembic/versions/c494649637d5_add_data_delivery_options.py rename to alembic/versions/2021_09_28_c494649637d5_add_data_delivery_options.py diff --git a/alembic/versions/f2edbd530656_convert_priority_to_enum.py b/alembic/versions/2021_12_16_f2edbd530656_convert_priority_to_enum.py similarity index 100% rename from alembic/versions/f2edbd530656_convert_priority_to_enum.py rename to alembic/versions/2021_12_16_f2edbd530656_convert_priority_to_enum.py diff --git a/alembic/versions/c76d655c8edf_add_fastq_qc_analysis_option_to_data_.py b/alembic/versions/2022_01_28_c76d655c8edf_add_fastq_qc_analysis_option_to_data_.py similarity index 100% rename from alembic/versions/c76d655c8edf_add_fastq_qc_analysis_option_to_data_.py rename to alembic/versions/2022_01_28_c76d655c8edf_add_fastq_qc_analysis_option_to_data_.py diff --git a/alembic/versions/76e8252a6efb_add_data_delivery_statina.py b/alembic/versions/2022_02_10_76e8252a6efb_add_data_delivery_statina.py similarity index 100% rename from alembic/versions/76e8252a6efb_add_data_delivery_statina.py rename to alembic/versions/2022_02_10_76e8252a6efb_add_data_delivery_statina.py diff --git a/alembic/versions/bf97b0121538_remove_avatars.py b/alembic/versions/2022_02_25_bf97b0121538_remove_avatars.py similarity index 100% rename from alembic/versions/bf97b0121538_remove_avatars.py rename to alembic/versions/2022_02_25_bf97b0121538_remove_avatars.py diff --git a/alembic/versions/0baf8309d227_clinical_trials.py b/alembic/versions/2022_03_28_0baf8309d227_clinical_trials.py similarity index 100% rename from alembic/versions/0baf8309d227_clinical_trials.py rename to alembic/versions/2022_03_28_0baf8309d227_clinical_trials.py diff --git a/alembic/versions/367813f2e597_support_of_1508_26.py b/alembic/versions/2022_04_06_367813f2e597_support_of_1508_26.py similarity index 100% rename from alembic/versions/367813f2e597_support_of_1508_26.py rename to alembic/versions/2022_04_06_367813f2e597_support_of_1508_26.py diff --git a/alembic/versions/33cd4b45acb4_add_data_analysis_option.py b/alembic/versions/2022_06_28_33cd4b45acb4_add_data_analysis_option.py similarity index 100% rename from alembic/versions/33cd4b45acb4_add_data_analysis_option.py rename to alembic/versions/2022_06_28_33cd4b45acb4_add_data_analysis_option.py diff --git a/alembic/versions/ddc94088be4d_adds_customer_to_customer_group_link_.py b/alembic/versions/2022_06_29_ddc94088be4d_adds_customer_to_customer_group_link_.py similarity index 100% rename from alembic/versions/ddc94088be4d_adds_customer_to_customer_group_link_.py rename to alembic/versions/2022_06_29_ddc94088be4d_adds_customer_to_customer_group_link_.py diff --git a/alembic/versions/2968d39ac35f_changes_in_delivery_options.py b/alembic/versions/2022_07_19_2968d39ac35f_changes_in_delivery_options.py similarity index 100% rename from alembic/versions/2968d39ac35f_changes_in_delivery_options.py rename to alembic/versions/2022_07_19_2968d39ac35f_changes_in_delivery_options.py diff --git a/alembic/versions/20750539a335_add_ticket_to_families.py b/alembic/versions/2022_07_22_20750539a335_add_ticket_to_families.py similarity index 100% rename from alembic/versions/20750539a335_add_ticket_to_families.py rename to alembic/versions/2022_07_22_20750539a335_add_ticket_to_families.py diff --git a/alembic/versions/9c9ca9407227_add_data_analysis_pon_option.py b/alembic/versions/2022_07_22_9c9ca9407227_add_data_analysis_pon_option.py similarity index 100% rename from alembic/versions/9c9ca9407227_add_data_analysis_pon_option.py rename to alembic/versions/2022_07_22_9c9ca9407227_add_data_analysis_pon_option.py diff --git a/alembic/versions/554140bc13e4_add_rnafusion_analysis_option.py b/alembic/versions/2022_09_27_554140bc13e4_add_rnafusion_analysis_option.py similarity index 100% rename from alembic/versions/554140bc13e4_add_rnafusion_analysis_option.py rename to alembic/versions/2022_09_27_554140bc13e4_add_rnafusion_analysis_option.py diff --git a/alembic/versions/df1b3dd317d0_add_trusted_customer_field.py b/alembic/versions/2023_04_12_df1b3dd317d0_add_trusted_customer_field.py similarity index 100% rename from alembic/versions/df1b3dd317d0_add_trusted_customer_field.py rename to alembic/versions/2023_04_12_df1b3dd317d0_add_trusted_customer_field.py diff --git a/alembic/versions/9008aa5065b4_add_taxprofiler_analysis_option.py b/alembic/versions/2023_04_19_9008aa5065b4_add_taxprofiler_analysis_option.py similarity index 100% rename from alembic/versions/9008aa5065b4_add_taxprofiler_analysis_option.py rename to alembic/versions/2023_04_19_9008aa5065b4_add_taxprofiler_analysis_option.py diff --git a/alembic/versions/ea5470295689_sequencing_stats.py b/alembic/versions/2023_05_04_ea5470295689_sequencing_stats.py similarity index 100% rename from alembic/versions/ea5470295689_sequencing_stats.py rename to alembic/versions/2023_05_04_ea5470295689_sequencing_stats.py diff --git a/alembic/versions/f5e0db62a5a7_update_sequencingstatistics.py b/alembic/versions/2023_05_26_f5e0db62a5a7_update_sequencingstatistics.py similarity index 100% rename from alembic/versions/f5e0db62a5a7_update_sequencingstatistics.py rename to alembic/versions/2023_05_26_f5e0db62a5a7_update_sequencingstatistics.py diff --git a/alembic/versions/e6a3f1ad4b50_change_read_count_to_bigint.py b/alembic/versions/2023_06_01_e6a3f1ad4b50_change_read_count_to_bigint.py similarity index 100% rename from alembic/versions/e6a3f1ad4b50_change_read_count_to_bigint.py rename to alembic/versions/2023_06_01_e6a3f1ad4b50_change_read_count_to_bigint.py diff --git a/alembic/versions/367ed257e4ee_change_data_type_of_metrics.py b/alembic/versions/2023_06_02_367ed257e4ee_change_data_type_of_metrics.py similarity index 100% rename from alembic/versions/367ed257e4ee_change_data_type_of_metrics.py rename to alembic/versions/2023_06_02_367ed257e4ee_change_data_type_of_metrics.py diff --git a/alembic/versions/869d352cc614_add_relationship_between_metrics_and_.py b/alembic/versions/2023_06_02_869d352cc614_add_relationship_between_metrics_and_.py similarity index 100% rename from alembic/versions/869d352cc614_add_relationship_between_metrics_and_.py rename to alembic/versions/2023_06_02_869d352cc614_add_relationship_between_metrics_and_.py diff --git a/alembic/versions/a34039f530c5_alter_column_types_sample_lane_.py b/alembic/versions/2023_06_02_a34039f530c5_alter_column_types_sample_lane_.py similarity index 100% rename from alembic/versions/a34039f530c5_alter_column_types_sample_lane_.py rename to alembic/versions/2023_06_02_a34039f530c5_alter_column_types_sample_lane_.py diff --git a/alembic/versions/97ffd22d7ebc_add_metrics_unique_constraint.py b/alembic/versions/2023_06_27_97ffd22d7ebc_add_metrics_unique_constraint.py similarity index 100% rename from alembic/versions/97ffd22d7ebc_add_metrics_unique_constraint.py rename to alembic/versions/2023_06_27_97ffd22d7ebc_add_metrics_unique_constraint.py diff --git a/alembic/versions/ffb9f8ab8e62_add_temp_read_col.py b/alembic/versions/2023_06_29_ffb9f8ab8e62_add_temp_read_col.py similarity index 100% rename from alembic/versions/ffb9f8ab8e62_add_temp_read_col.py rename to alembic/versions/2023_06_29_ffb9f8ab8e62_add_temp_read_col.py diff --git a/alembic/versions/c3fdf3a8a5b3_add_data_archive_location.py b/alembic/versions/2023_07_04_c3fdf3a8a5b3_add_data_archive_location.py similarity index 100% rename from alembic/versions/c3fdf3a8a5b3_add_data_archive_location.py rename to alembic/versions/2023_07_04_c3fdf3a8a5b3_add_data_archive_location.py diff --git a/alembic/versions/68e54d17f4f3_add_is_clinical_to_customer.py b/alembic/versions/2023_07_07_68e54d17f4f3_add_is_clinical_to_customer.py similarity index 100% rename from alembic/versions/68e54d17f4f3_add_is_clinical_to_customer.py rename to alembic/versions/2023_07_07_68e54d17f4f3_add_is_clinical_to_customer.py diff --git a/alembic/versions/2fdb42ba801a_support_novaseqx_flow_cell_table.py b/alembic/versions/2023_07_10_2fdb42ba801a_support_novaseqx_flow_cell_table.py similarity index 100% rename from alembic/versions/2fdb42ba801a_support_novaseqx_flow_cell_table.py rename to alembic/versions/2023_07_10_2fdb42ba801a_support_novaseqx_flow_cell_table.py diff --git a/alembic/versions/201b16c45366_drop_calc_reads_columns.py b/alembic/versions/2023_08_01_201b16c45366_drop_calc_reads_columns.py similarity index 100% rename from alembic/versions/201b16c45366_drop_calc_reads_columns.py rename to alembic/versions/2023_08_01_201b16c45366_drop_calc_reads_columns.py diff --git a/alembic/versions/5cb3ce4c3e39_rename_q30_metrics_column.py b/alembic/versions/2023_08_18_5cb3ce4c3e39_rename_q30_metrics_column.py similarity index 100% rename from alembic/versions/5cb3ce4c3e39_rename_q30_metrics_column.py rename to alembic/versions/2023_08_18_5cb3ce4c3e39_rename_q30_metrics_column.py diff --git a/alembic/versions/9def7a6eae73_sequenced_at_to_reads_updated_at.py b/alembic/versions/2023_09_04_9def7a6eae73_sequenced_at_to_reads_updated_at.py similarity index 100% rename from alembic/versions/9def7a6eae73_sequenced_at_to_reads_updated_at.py rename to alembic/versions/2023_09_04_9def7a6eae73_sequenced_at_to_reads_updated_at.py diff --git a/alembic/versions/936887038ba8_sync_cg_and_statusdb.py b/alembic/versions/2023_09_15_936887038ba8_sync_cg_and_statusdb.py similarity index 100% rename from alembic/versions/936887038ba8_sync_cg_and_statusdb.py rename to alembic/versions/2023_09_15_936887038ba8_sync_cg_and_statusdb.py diff --git a/alembic/versions/d0aa961845b9_.py b/alembic/versions/2023_09_19_d0aa961845b9_sync_naming.py similarity index 100% rename from alembic/versions/d0aa961845b9_.py rename to alembic/versions/2023_09_19_d0aa961845b9_sync_naming.py diff --git a/alembic/versions/c3da223e60d8_add_has_backup_field.py b/alembic/versions/2023_09_22_c3da223e60d8_add_has_backup_field.py similarity index 100% rename from alembic/versions/c3da223e60d8_add_has_backup_field.py rename to alembic/versions/2023_09_22_c3da223e60d8_add_has_backup_field.py diff --git a/alembic/versions/e853d21feaa0_add_pipeline_limitations_table.py b/alembic/versions/2023_09_28_e853d21feaa0_add_pipeline_limitations_table.py similarity index 100% rename from alembic/versions/e853d21feaa0_add_pipeline_limitations_table.py rename to alembic/versions/2023_09_28_e853d21feaa0_add_pipeline_limitations_table.py diff --git a/alembic/versions/db61c62d9bc0_add_customer_lab_contact.py b/alembic/versions/2023_10_05_db61c62d9bc0_add_customer_lab_contact.py similarity index 100% rename from alembic/versions/db61c62d9bc0_add_customer_lab_contact.py rename to alembic/versions/2023_10_05_db61c62d9bc0_add_customer_lab_contact.py diff --git a/alembic/versions/392e49db40fc_add_sample_concentration_min_max.py b/alembic/versions/2023_10_25_392e49db40fc_add_sample_concentration_min_max.py similarity index 100% rename from alembic/versions/392e49db40fc_add_sample_concentration_min_max.py rename to alembic/versions/2023_10_25_392e49db40fc_add_sample_concentration_min_max.py diff --git a/alembic/versions/9073c61bc72b_add_raredisease_to_analysis_options.py b/alembic/versions/2023_10_31_9073c61bc72b_add_raredisease_to_analysis_options.py similarity index 100% rename from alembic/versions/9073c61bc72b_add_raredisease_to_analysis_options.py rename to alembic/versions/2023_10_31_9073c61bc72b_add_raredisease_to_analysis_options.py diff --git a/alembic/versions/9b188aee9577_add_new_tech_tables.py b/alembic/versions/2024_05_03_9b188aee9577_add_new_tech_tables.py similarity index 100% rename from alembic/versions/9b188aee9577_add_new_tech_tables.py rename to alembic/versions/2024_05_03_9b188aee9577_add_new_tech_tables.py diff --git a/alembic/versions/5fd7e8758fb1_add_autoincrement_to_device_tables.py b/alembic/versions/2024_05_21_5fd7e8758fb1_add_autoincrement_to_device_tables.py similarity index 100% rename from alembic/versions/5fd7e8758fb1_add_autoincrement_to_device_tables.py rename to alembic/versions/2024_05_21_5fd7e8758fb1_add_autoincrement_to_device_tables.py From 8fe98c62da204d332820a540791df8edb88a5428 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 23 May 2024 08:21:29 +0000 Subject: [PATCH 36/42] =?UTF-8?q?Bump=20version:=2060.8.3=20=E2=86=92=2060?= =?UTF-8?q?.8.4=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8038f35067..b4d08f814a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.8.3 +current_version = 60.8.4 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 56148e35e9..1bf687d9d3 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.8.3" +__version__ = "60.8.4" diff --git a/pyproject.toml b/pyproject.toml index a5a455dfa1..420ef93762 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.8.3" +version = "60.8.4" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 60745f33f8db5d3f01a9c42a82ff06ace5ae9d20 Mon Sep 17 00:00:00 2001 From: ChristianOertlin Date: Thu, 23 May 2024 10:25:56 +0200 Subject: [PATCH 37/42] add(yield columns to new sample run metrics) (#3264) (patch) # Description refactoring of tables and adding new columns to run device project tables --- ...24_05_22_18e3b2aba252_add_yield_columns.py | 84 +++++++++++++++++++ .../bcl_convert_metrics_service.py | 6 +- cg/store/models.py | 34 ++++---- 3 files changed, 106 insertions(+), 18 deletions(-) create mode 100644 alembic/versions/2024_05_22_18e3b2aba252_add_yield_columns.py diff --git a/alembic/versions/2024_05_22_18e3b2aba252_add_yield_columns.py b/alembic/versions/2024_05_22_18e3b2aba252_add_yield_columns.py new file mode 100644 index 0000000000..1373b83333 --- /dev/null +++ b/alembic/versions/2024_05_22_18e3b2aba252_add_yield_columns.py @@ -0,0 +1,84 @@ +"""2024_05_22_add_yield_columns + +Revision ID: 18e3b2aba252 +Revises: 5fd7e8758fb1 +Create Date: 2024-05-22 14:01:15.197675 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "18e3b2aba252" +down_revision = "5fd7e8758fb1" +branch_labels = None +depends_on = None + + +def upgrade(): + + # rename table IlluminaSampleRunMetrics to IlluminaSampleSequencingMetrics + op.rename_table( + old_table_name="illumina_sample_run_metrics", + new_table_name="illumina_sample_sequencing_metrics", + ) + # add column yield + op.add_column( + table_name="illumina_sample_sequencing_metrics", + column=sa.Column("yield", sa.Integer(), nullable=True), + ) + # add column yield_q30 + op.add_column( + table_name="illumina_sample_sequencing_metrics", + column=sa.Column("yield_q30", sa.Float(), nullable=True), + ) + # rename table run metrics to InstrumentRun + op.rename_table( + old_table_name="run_metrics", + new_table_name="instrument_run", + ) + # on SampleRunMetrics change run_metrics_id to instrument_run_id + op.alter_column( + table_name="sample_run_metrics", + column_name="run_metrics_id", + new_column_name="instrument_run_id", + type_=sa.Integer(), + ) + # rename illumina_sequencing_metrics to illumina_sequencing_run + op.rename_table( + old_table_name="illumina_sequencing_metrics", + new_table_name="illumina_sequencing_run", + ) + + +def downgrade(): + # rename illumina_sequencing_metrics to illumina_sequencing_run + op.rename_table( + new_table_name="illumina_sequencing_metrics", + old_table_name="illumina_sequencing_run", + ) + + # on SampleRunMetrics change run_metrics_id to instrument_run_id + op.alter_column( + table_name="sample_run_metrics", + new_column_name="run_metrics_id", + column_name="instrument_run_id", + type_=sa.Integer(), + ) + # rename run metrics -> InstrumentRun + op.rename_table( + new_table_name="run_metrics", + old_table_name="instrument_run", + ) + + # drop columns yield and yield_q30 + op.drop_column(table_name="illumina_sample_sequencing_metrics", column_name="yield") + op.drop_column(table_name="illumina_sample_sequencing_metrics", column_name="yield_q30") + + # rename table IlluminaSampleSequencingMetrics to IlluminaSampleRunMetrics + op.rename_table( + new_table_name="illumina_sample_run_metrics", + old_table_name="illumina_sample_sequencing_metrics", + ) diff --git a/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py b/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py index c18780b7be..ab3f186df3 100644 --- a/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py +++ b/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py @@ -6,7 +6,7 @@ from cg.constants.devices import DeviceType from cg.models.flow_cell.flow_cell import FlowCellDirectoryData from cg.services.bcl_convert_metrics_service.parser import MetricsParser -from cg.store.models import SampleLaneSequencingMetrics, IlluminaSampleRunMetrics +from cg.store.models import SampleLaneSequencingMetrics, IlluminaSampleSequencingMetrics from cg.store.store import Store from cg.utils.flow_cell import get_flow_cell_id @@ -87,7 +87,7 @@ def create_sample_run_metrics( run_metrics_id: int, metrics_parser: MetricsParser, store: Store, - ) -> IlluminaSampleRunMetrics: + ) -> IlluminaSampleSequencingMetrics: """Create sequencing metrics for all lanes in a flow cell.""" total_reads: int = metrics_parser.calculate_total_reads_for_sample_in_lane( @@ -101,7 +101,7 @@ def create_sample_run_metrics( ) sample_id: int = store.get_sample_by_internal_id(sample_internal_id).id - return IlluminaSampleRunMetrics( + return IlluminaSampleSequencingMetrics( run_metrics_id=run_metrics_id, sample_id=sample_id, type=DeviceType.ILLUMINA, diff --git a/cg/store/models.py b/cg/store/models.py index 0f2475b2cd..6ae3d8e514 100644 --- a/cg/store/models.py +++ b/cg/store/models.py @@ -961,7 +961,7 @@ def to_dict(self): class RunDevice(Base): - """Model for storing run devices.""" + """Parent model for the different types of run devices.""" __tablename__ = "run_device" @@ -969,7 +969,7 @@ class RunDevice(Base): type: Mapped[DeviceType] internal_id: Mapped[UniqueStr64] - run_metrics: Mapped[list["RunMetrics"]] = orm.relationship( + instrument_runs: Mapped[list["InstrumentRun"]] = orm.relationship( back_populates="device", cascade="all, delete" ) @@ -979,7 +979,7 @@ def _samples(self) -> list[Sample]: return list( { sample_run_metric.sample - for run in self.run_metrics + for run in self.instrument_run for sample_run_metric in run.sample_run_metrics } ) @@ -1002,18 +1002,18 @@ class IlluminaFlowCell(RunDevice): __mapper_args__ = {"polymorphic_identity": DeviceType.ILLUMINA} -class RunMetrics(Base): - """Model for storing run devices.""" +class InstrumentRun(Base): + """Parent model for the different types of instrument runs.""" - __tablename__ = "run_metrics" + __tablename__ = "instrument_run" id: Mapped[PrimaryKeyInt] type: Mapped[DeviceType] device_id: Mapped[int] = mapped_column(ForeignKey("run_device.id")) - device: Mapped[RunDevice] = orm.relationship(back_populates="run_metrics") + device: Mapped[RunDevice] = orm.relationship(back_populates="instrument_runs") sample_metrics: Mapped[list["SampleRunMetrics"]] = orm.relationship( - back_populates="run_metrics", cascade="all, delete" + back_populates="instrument_run", cascade="all, delete" ) __mapper_args__ = { @@ -1021,10 +1021,10 @@ class RunMetrics(Base): } -class IlluminaRunMetrics(RunMetrics): - __tablename__ = "illumina_sequencing_metrics" +class IlluminaSequencingRun(InstrumentRun): + __tablename__ = "illumina_sequencing_run" - id: Mapped[int] = mapped_column(ForeignKey("run_metrics.id"), primary_key=True) + id: Mapped[int] = mapped_column(ForeignKey("instrument_run.id"), primary_key=True) sequencer_type: Mapped[str | None] = mapped_column( types.Enum("hiseqga", "hiseqx", "novaseq", "novaseqx") ) @@ -1053,13 +1053,15 @@ class IlluminaRunMetrics(RunMetrics): class SampleRunMetrics(Base): + """Parent model for the different types of sample run metrics.""" + __tablename__ = "sample_run_metrics" id: Mapped[PrimaryKeyInt] sample_id: Mapped[int] = mapped_column(ForeignKey("sample.id")) - run_metrics_id: Mapped[int] = mapped_column(ForeignKey("run_metrics.id")) + instrument_run_id: Mapped[int] = mapped_column(ForeignKey("instrument_run.id")) type: Mapped[DeviceType] - run_metrics: Mapped[RunMetrics] = orm.relationship(back_populates="sample_metrics") + instrument_run: Mapped[InstrumentRun] = orm.relationship(back_populates="sample_metrics") sample: Mapped[Sample] = orm.relationship(back_populates="_new_run_metrics") __mapper_args__ = { @@ -1067,15 +1069,17 @@ class SampleRunMetrics(Base): } -class IlluminaSampleRunMetrics(SampleRunMetrics): +class IlluminaSampleSequencingMetrics(SampleRunMetrics): """Sequencing metrics for a sample sequenced on an Illumina instrument. The metrics are per sample, per lane, per flow cell.""" - __tablename__ = "illumina_sample_run_metrics" + __tablename__ = "illumina_sample_sequencing_metrics" id: Mapped[int] = mapped_column(ForeignKey("sample_run_metrics.id"), primary_key=True) flow_cell_lane: Mapped[int | None] total_reads_in_lane: Mapped[BigInt | None] base_passing_q30_percent: Mapped[Num_6_2 | None] base_mean_quality_score: Mapped[Num_6_2 | None] + _yield: Mapped[BigInt | None] = mapped_column("yield", quote=True) + yield_q30: Mapped[Num_6_2 | None] created_at: Mapped[datetime | None] __mapper_args__ = {"polymorphic_identity": DeviceType.ILLUMINA} From 7066d3fb55b451c7fd54c3dba0d709b8462ab2cc Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 23 May 2024 08:26:22 +0000 Subject: [PATCH 38/42] =?UTF-8?q?Bump=20version:=2060.8.4=20=E2=86=92=2060?= =?UTF-8?q?.8.5=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b4d08f814a..c55ff83bbf 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.8.4 +current_version = 60.8.5 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 1bf687d9d3..bd405250d4 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.8.4" +__version__ = "60.8.5" diff --git a/pyproject.toml b/pyproject.toml index 420ef93762..20293eac9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.8.4" +version = "60.8.5" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From e3739c0c87e3ac07f14b2fdf930e70b835961d47 Mon Sep 17 00:00:00 2001 From: ChristianOertlin Date: Thu, 23 May 2024 11:19:47 +0200 Subject: [PATCH 39/42] refactor(demux post processing run parameters) (#3263) (patch) # Descripion Refactoring of the run parameters and extraction of flow cell model from run paramters. --- cg/constants/metrics.py | 2 + cg/models/demultiplex/run_parameters.py | 39 +++++++++++++---- .../bcl_convert_metrics_service.py | 23 ++++++++++ .../illumina_post_processing_service.py | 2 +- .../illumina_post_processing_service/utils.py | 13 ------ cg/store/crud/create.py | 7 +++ .../run_parameters_fixtures.py | 4 +- .../demultiplexing/test_run_parameters.py | 28 +++++++++++- .../conftest.py | 26 ----------- .../test_illumina_post_processing_utils.py | 43 ------------------- 10 files changed, 93 insertions(+), 94 deletions(-) delete mode 100644 tests/services/illumina_post_processing_service/test_illumina_post_processing_utils.py diff --git a/cg/constants/metrics.py b/cg/constants/metrics.py index 7a4b320a20..a85e443b26 100644 --- a/cg/constants/metrics.py +++ b/cg/constants/metrics.py @@ -10,6 +10,8 @@ class QualityMetricsColumnNames(StrEnum): SAMPLE_INTERNAL_ID: str = "SampleID" MEAN_QUALITY_SCORE_Q30: str = "Mean Quality Score (PF)" Q30_BASES_PERCENT: str = "% Q30" + YIELD: str = "Yield" + YIELD_Q30: str = "YieldQ30" class DemuxMetricsColumnNames(StrEnum): diff --git a/cg/models/demultiplex/run_parameters.py b/cg/models/demultiplex/run_parameters.py index 4bb9143e67..87c84f1bf0 100644 --- a/cg/models/demultiplex/run_parameters.py +++ b/cg/models/demultiplex/run_parameters.py @@ -3,7 +3,7 @@ import logging from abc import abstractmethod from pathlib import Path -from xml.etree import ElementTree +from xml.etree.ElementTree import Element, ElementTree from packaging.version import parse @@ -44,9 +44,7 @@ def _validate_instrument(self, node_name: str, node_value: str): Raises: RunParametersError if the node does not have the expected value.""" try: - application: ElementTree.Element | None = get_tree_node( - tree=self.tree, node_name=node_name - ) + application: Element | None = get_tree_node(tree=self.tree, node_name=node_name) except XMLError: raise RunParametersError( f"Could not find node {node_name} in the run parameters file. " @@ -129,6 +127,10 @@ def _get_index_settings(self) -> IndexSettings: return NOVASEQ_6000_POST_1_5_KITS_INDEX_SETTINGS return NO_REVERSE_COMPLEMENTS_INDEX_SETTINGS + @abstractmethod + def get_flow_cell_model(self): + pass + def __str__(self): return f"RunParameters(path={self.path}, sequencer={self.sequencer})" @@ -189,6 +191,10 @@ def get_read_2_cycles(self) -> int: node_name: str = RunParametersXMLNodes.READ_2_HISEQ return self._get_node_integer_value(node_name=node_name) + def get_flow_cell_model(self) -> None: + """Return None for run parameters associated with HiSeq sequencing.""" + return None + class RunParametersNovaSeq6000(RunParameters): """Specific class for parsing run parameters of NovaSeq6000 sequencing.""" @@ -210,7 +216,7 @@ def control_software_version(self) -> str: def reagent_kit_version(self) -> str: """Return the reagent kit version if existent, return 'unknown' otherwise.""" node_name: str = RunParametersXMLNodes.REAGENT_KIT_VERSION - xml_node: ElementTree.Element | None = self.tree.find(node_name) + xml_node: Element | None = self.tree.find(node_name) if xml_node is None: LOG.warning("Could not determine reagent kit version") LOG.info("Set reagent kit version to 'unknown'") @@ -242,6 +248,11 @@ def get_read_2_cycles(self) -> int: node_name: str = RunParametersXMLNodes.READ_2_NOVASEQ_6000 return self._get_node_integer_value(node_name=node_name) + def get_flow_cell_model(self) -> str: + """Return the flow cell model referred to as 'FlowCellMode' in the run parameters file.""" + node_name: str = RunParametersXMLNodes.FLOW_CELL_MODE + return self._get_node_string_value(node_name=node_name) + class RunParametersNovaSeqX(RunParameters): """Specific class for parsing run parameters of NovaSeqX sequencing.""" @@ -272,12 +283,10 @@ def sequencer(self) -> str: def _read_parser(self) -> dict[str, int]: """Return read and index cycle values parsed as a dictionary.""" cycle_mapping: dict[str, int] = {} - planned_reads_tree: ElementTree.Element = get_tree_node( + planned_reads_tree: Element = get_tree_node( tree=self.tree, node_name=RunParametersXMLNodes.PLANNED_READS_NOVASEQ_X ) - planned_reads: list[ElementTree.Element] = planned_reads_tree.findall( - RunParametersXMLNodes.INNER_READ - ) + planned_reads: list[Element] = planned_reads_tree.findall(RunParametersXMLNodes.INNER_READ) for read_elem in planned_reads: read_name: str = read_elem.get(RunParametersXMLNodes.READ_NAME) cycles: int = int(read_elem.get(RunParametersXMLNodes.CYCLES)) @@ -299,3 +308,15 @@ def get_read_1_cycles(self) -> int: def get_read_2_cycles(self) -> int: """Return the number of cycles in the second read.""" return self._read_parser.get(RunParametersXMLNodes.READ_2_NOVASEQ_X) + + def get_flow_cell_model(self) -> str: + """Return the flow cell model referred to as 'Mode' or 'Name' in the run parameters file.""" + consumable_infos: list[Element] = self.tree.findall(".//ConsumableInfo") + for consumable_info in consumable_infos: + type_element: Element | None = consumable_info.find("Type") + if type_element is not None and type_element.text == "FlowCell": + name_element: Element | None = consumable_info.find("Name") or consumable_info.find( + "Mode" + ) + if name_element is not None: + return name_element.text diff --git a/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py b/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py index ab3f186df3..11724d01fb 100644 --- a/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py +++ b/cg/services/bcl_convert_metrics_service/bcl_convert_metrics_service.py @@ -111,3 +111,26 @@ def create_sample_run_metrics( base_mean_quality_score=mean_quality_score, created_at=datetime.now(), ) + + def create_sample_sequencing_metrics_for_flow_cell( + self, + flow_cell_directory: Path, + run_metrics_id: int, + store: Store, + ) -> list[IlluminaSampleSequencingMetrics]: + """Parse the demultiplexing metrics data into the sequencing statistics model.""" + metrics_parser = MetricsParser(flow_cell_directory) + sample_internal_ids: list[str] = metrics_parser.get_sample_internal_ids() + sample_lane_sequencing_metrics: list[IlluminaSampleSequencingMetrics] = [] + + for sample_internal_id in sample_internal_ids: + for lane in metrics_parser.get_lanes_for_sample(sample_internal_id=sample_internal_id): + metrics: IlluminaSampleSequencingMetrics = self.create_sample_run_metrics( + sample_internal_id=sample_internal_id, + lane=lane, + metrics_parser=metrics_parser, + run_metrics_id=run_metrics_id, + store=store, + ) + sample_lane_sequencing_metrics.append(metrics) + return sample_lane_sequencing_metrics diff --git a/cg/services/illumina_post_processing_service/illumina_post_processing_service.py b/cg/services/illumina_post_processing_service/illumina_post_processing_service.py index 3b02987437..8f49724ac3 100644 --- a/cg/services/illumina_post_processing_service/illumina_post_processing_service.py +++ b/cg/services/illumina_post_processing_service/illumina_post_processing_service.py @@ -36,7 +36,7 @@ def store_illumina_flow_cell( Create flow cell from the parsed and validated flow cell data. And add the samples on the flow cell to the model. """ - model: str | None = get_flow_cell_model_from_run_parameters(flow_cell.run_parameters_path) + model: str | None = flow_cell.run_parameters.get_flow_cell_model() new_flow_cell = IlluminaFlowCell( internal_id=flow_cell.id, type=DeviceType.ILLUMINA, model=model ) diff --git a/cg/services/illumina_post_processing_service/utils.py b/cg/services/illumina_post_processing_service/utils.py index 2b2874dd7a..f070fdbca0 100644 --- a/cg/services/illumina_post_processing_service/utils.py +++ b/cg/services/illumina_post_processing_service/utils.py @@ -10,16 +10,3 @@ def create_delivery_file_in_flow_cell_directory(flow_cell_directory: Path) -> None: Path(flow_cell_directory, DemultiplexingDirsAndFiles.DELIVERY).touch() - - -def get_flow_cell_model_from_run_parameters(run_parameters_path: Path) -> str | None: - """Return the model of the flow cell.""" - xml_tree: ElementTree = read_xml(run_parameters_path) - node: Element | None = None - for node_name in [RunParametersXMLNodes.MODE, RunParametersXMLNodes.FLOW_CELL_MODE]: - try: - node: Element = get_tree_node(xml_tree, node_name) - return node.text - except XMLError: - continue - return node diff --git a/cg/store/crud/create.py b/cg/store/crud/create.py index addc51a051..b8920f5fd9 100644 --- a/cg/store/crud/create.py +++ b/cg/store/crud/create.py @@ -31,6 +31,7 @@ User, order_case, IlluminaFlowCell, + IlluminaSequencingRun, ) LOG = logging.getLogger(__name__) @@ -425,3 +426,9 @@ def add_illumina_flow_cell(self, flow_cell: IlluminaFlowCell) -> IlluminaFlowCel session.add(flow_cell) LOG.debug(f"Flow cell added to status db: {flow_cell.id}.") return flow_cell + + def add_illumina_sequencing_metrics( + self, sequencing_metrics: IlluminaSequencingRun + ) -> IlluminaSequencingRun: + """Add a new Illumina flow cell to the status database as a pending transaction.""" + pass diff --git a/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py b/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py index 32b92c8b00..f5694f15a4 100644 --- a/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py +++ b/tests/fixture_plugins/demultiplex_fixtures/run_parameters_fixtures.py @@ -26,7 +26,9 @@ def run_parameters_novaseq_6000_different_index( @pytest.fixture(scope="function") -def run_parameters_novaseq_x_different_index(run_parameters_dir: Path) -> RunParametersNovaSeqX: +def run_parameters_novaseq_x_different_index( + run_parameters_dir: Path, +) -> RunParametersNovaSeqX: """Return a NovaSeqX RunParameters object with different index cycles.""" path = Path(run_parameters_dir, "RunParameters_novaseq_X_different_index_cycles.xml") return RunParametersNovaSeqX(run_parameters_path=path) diff --git a/tests/models/demultiplexing/test_run_parameters.py b/tests/models/demultiplexing/test_run_parameters.py index dad052215b..d99bbf072d 100644 --- a/tests/models/demultiplexing/test_run_parameters.py +++ b/tests/models/demultiplexing/test_run_parameters.py @@ -51,7 +51,11 @@ def test_run_parameters_parent_class_fails( "run_parameters_path, constructor, sequencer", [ ("hiseq_x_single_index_run_parameters_path", RunParametersHiSeq, Sequencers.HISEQX), - ("hiseq_2500_dual_index_run_parameters_path", RunParametersHiSeq, Sequencers.HISEQGA), + ( + "hiseq_2500_dual_index_run_parameters_path", + RunParametersHiSeq, + Sequencers.HISEQGA, + ), ( "novaseq_6000_run_parameters_pre_1_5_kits_path", RunParametersNovaSeq6000, @@ -301,3 +305,25 @@ def test_get_index_settings( settings: IndexSettings = flow_cell.run_parameters.index_settings # THEN the correct index settings are returned assert settings == correct_settings + + +@pytest.mark.parametrize( + "run_parameters_fixture, expected_result", + [ + ("hiseq_x_single_index_run_parameters", None), + ("hiseq_2500_dual_index_run_parameters", None), + ("novaseq_6000_run_parameters_pre_1_5_kits", "S4"), + ("novaseq_6000_run_parameters_post_1_5_kits", "S1"), + ("novaseq_x_run_parameters", "10B"), + ], +) +def test_get_flow_cell_mode( + run_parameters_fixture: str, expected_result: str | None, request: FixtureRequest +): + """Test that the correct flow cell mode is returned for each RunParameters object.""" + # GIVEN a RunParameters object + run_parameters: RunParameters = request.getfixturevalue(run_parameters_fixture) + # WHEN getting the flow cell mode + result: str | None = run_parameters.get_flow_cell_model() + # THEN the correct flow cell mode is returned + assert result == expected_result diff --git a/tests/services/illumina_post_processing_service/conftest.py b/tests/services/illumina_post_processing_service/conftest.py index f530692a81..1c27c857de 100644 --- a/tests/services/illumina_post_processing_service/conftest.py +++ b/tests/services/illumina_post_processing_service/conftest.py @@ -3,29 +3,3 @@ from pathlib import Path import pytest - - -@pytest.fixture -def run_paramters_with_flow_cell_mode_node_name() -> Path: - return Path( - "tests", - "fixtures", - "apps", - "demultiplexing", - "flow_cells", - "230912_A00187_1009_AHK33MDRX3", - "RunParameters.xml", - ) - - -@pytest.fixture -def run_parameters_without_model() -> Path: - return Path( - "tests", - "fixtures", - "apps", - "demultiplexing", - "flow_cells", - "170517_ST-E00266_0210_BHJCFFALXX", - "runParameters.xml", - ) diff --git a/tests/services/illumina_post_processing_service/test_illumina_post_processing_utils.py b/tests/services/illumina_post_processing_service/test_illumina_post_processing_utils.py deleted file mode 100644 index b4e6ed7cc7..0000000000 --- a/tests/services/illumina_post_processing_service/test_illumina_post_processing_utils.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Test utils of the illumina post processing service.""" - -from pathlib import Path - -from cg.services.illumina_post_processing_service.utils import ( - get_flow_cell_model_from_run_parameters, -) - - -def test_get_flow_cell_model_from_run_parameters_with_mode_node_name( - novaseq_x_run_parameters_path: Path, -): - # GIVEN a run parameters file from a NovaSeq X run - - # WHEN getting the flow cell model from the run parameters file - flow_cell_model = get_flow_cell_model_from_run_parameters(novaseq_x_run_parameters_path) - - # THEN the flow cell model should be returned - assert flow_cell_model == "10B" - - -def test_get_flow_cel_model_from_run_parameters_with_flow_cell_mode_node_name( - run_paramters_with_flow_cell_mode_node_name: Path, -): - # GIVEN a run parameters file with a FlowCellMode node - - # WHEN getting the flow cell model from the run parameters file - flow_cell_model = get_flow_cell_model_from_run_parameters( - run_paramters_with_flow_cell_mode_node_name - ) - - # THEN the flow cell model should be returned - assert flow_cell_model == "S1" - - -def test_get_flow_cell_model_from_run_paramters_without_model(run_parameters_without_model: Path): - # GIVEN a run parameters file without a model - - # WHEN getting the flow cell model from the run parameters file - flow_cell_model = get_flow_cell_model_from_run_parameters(run_parameters_without_model) - - # THEN the flow cell model should be None - assert flow_cell_model is None From 7772678d5fae08740966914bdc07d2f319ddfd6d Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 23 May 2024 09:20:12 +0000 Subject: [PATCH 40/42] =?UTF-8?q?Bump=20version:=2060.8.5=20=E2=86=92=2060?= =?UTF-8?q?.8.6=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c55ff83bbf..ba92429b0c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.8.5 +current_version = 60.8.6 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index bd405250d4..3e0d55218a 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.8.5" +__version__ = "60.8.6" diff --git a/pyproject.toml b/pyproject.toml index 20293eac9d..f5d920f3dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.8.5" +version = "60.8.6" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md" From 8a022a0435dbf203ef16e75da7f40b6bc63a1d7e Mon Sep 17 00:00:00 2001 From: Henrik Stranneheim Date: Thu, 23 May 2024 13:36:18 +0200 Subject: [PATCH 41/42] feat(update): requests (#3268) ### Changed - Update requests --- poetry.lock | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8c1db6ba57..1e2606f7d7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1964,7 +1964,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2001,13 +2000,13 @@ files = [ [[package]] name = "requests" -version = "2.31.0" +version = "2.32.2" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] @@ -2427,4 +2426,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "c2fe6ddb49e5c12ac17d3967ea5fda0bac1701dd2c6b117f99591a8f1b73ab0c" \ No newline at end of file +content-hash = "c2fe6ddb49e5c12ac17d3967ea5fda0bac1701dd2c6b117f99591a8f1b73ab0c" From 07d7456ed0a15c5f2565af1322063a56c6a627b5 Mon Sep 17 00:00:00 2001 From: Clinical Genomics Bot Date: Thu, 23 May 2024 11:36:46 +0000 Subject: [PATCH 42/42] =?UTF-8?q?Bump=20version:=2060.8.6=20=E2=86=92=2060?= =?UTF-8?q?.8.7=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- cg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ba92429b0c..159ac32d92 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 60.8.6 +current_version = 60.8.7 commit = True tag = True tag_name = v{new_version} diff --git a/cg/__init__.py b/cg/__init__.py index 3e0d55218a..ca941b04b6 100644 --- a/cg/__init__.py +++ b/cg/__init__.py @@ -1,2 +1,2 @@ __title__ = "cg" -__version__ = "60.8.6" +__version__ = "60.8.7" diff --git a/pyproject.toml b/pyproject.toml index f5d920f3dc..4d71a2e343 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "cg" -version = "60.8.6" +version = "60.8.7" description = "Clinical Genomics command center" authors = ["Clinical Genomics "] readme = "README.md"