feature(dev-branch-pacbio)

cg/apps/housekeeper/hk.py

@@ -7,11 +7,7 @@
 
 from housekeeper.include import checksum as hk_checksum
 from housekeeper.include import include_version
-from housekeeper.store.database import (
-    create_all_tables,
-    drop_all_tables,
-    initialize_database,
-)
+from housekeeper.store.database import create_all_tables, drop_all_tables, initialize_database
 from housekeeper.store.models import Archive, Bundle, File, Tag, Version
 from housekeeper.store.store import Store
 from sqlalchemy.orm import Query
@@ -570,19 +566,19 @@ def add_bundle_and_version_if_non_existent(self, bundle_name: str) -> None:
         else:
             LOG.debug(f"Bundle with name {bundle_name} already exists")
 
-    def store_fastq_path_in_housekeeper(
+    def create_bundle_and_add_file_with_tags(
         self,
-        sample_internal_id: str,
-        sample_fastq_path: Path,
-        flow_cell_id: str,
+        bundle_name: str,
+        file_path: Path,
+        tags: list[str],
     ) -> None:
-        """Add the fastq file path with tags to a bundle and version in Housekeeper."""
-        self.add_bundle_and_version_if_non_existent(sample_internal_id)
-        self.add_tags_if_non_existent([sample_internal_id])
+        """Add a file path with tags to a bundle and version in Housekeeper."""
+        self.add_bundle_and_version_if_non_existent(bundle_name)
+        self.add_tags_if_non_existent([bundle_name])
         self.add_file_to_bundle_if_non_existent(
-            file_path=sample_fastq_path,
-            bundle_name=sample_internal_id,
-            tag_names=[SequencingFileTag.FASTQ, flow_cell_id, sample_internal_id],
+            file_path=file_path,
+            bundle_name=bundle_name,
+            tag_names=tags,
         )
 
     def get_archive_entries(

cg/cli/base.py

@@ -9,8 +9,6 @@
 from sqlalchemy.orm import scoped_session
 
 import cg
-from cg.cli.sequencing_qc.sequencing_qc import sequencing_qc
-from cg.cli.validate import validate
 from cg.cli.add import add as add_cmd
 from cg.cli.archive import archive
 from cg.cli.backup import backup
@@ -22,6 +20,8 @@
 from cg.cli.downsample import downsample
 from cg.cli.generate.base import generate as generate_cmd
 from cg.cli.get import get
+from cg.cli.post_process.post_process import post_process_group as post_processing
+from cg.cli.sequencing_qc.sequencing_qc import sequencing_qc
 from cg.cli.set.base import set_cmd
 from cg.cli.store.base import store as store_cmd
 from cg.cli.transfer import transfer_group
@@ -120,5 +120,6 @@ def init(context: CGConfig, reset: bool, force: bool):
 base.add_command(demultiplex_cmd)
 base.add_command(generate_cmd)
 base.add_command(downsample)
+base.add_command(post_processing)
 base.add_command(validate)
 base.add_command(sequencing_qc)

cg/cli/post_process/post_process.py

@@ -0,0 +1,39 @@
+"""CLI commands to post-process a sequencing run."""
+
+import logging
+
+import click
+
+from cg.cli.post_process.utils import get_post_processing_service_from_run_name
+from cg.cli.utils import CLICK_CONTEXT_SETTINGS
+from cg.constants.cli_options import DRY_RUN
+from cg.models.cg_config import CGConfig
+from cg.services.run_devices.abstract_classes import PostProcessingService
+
+LOG = logging.getLogger(__name__)
+
+
+@click.group(name="post-process", context_settings=CLICK_CONTEXT_SETTINGS)
+def post_process_group():
+    """Post-process sequencing runs from the sequencing instruments."""
+    LOG.info("Running cg post-processing.")
+
+
+@post_process_group.command(name="run")
+@DRY_RUN
+@click.argument("run-name")
+@click.pass_obj
+def post_process_sequencing_run(context: CGConfig, run_name: str, dry_run: bool):
+    """Post-process a sequencing run from the PacBio instrument.
+
+    run-name is the full name of the sequencing unit of run. For example:
+        PacBio: 'r84202_20240522_133539/1_A01'
+    """
+    post_processing_service: PostProcessingService = get_post_processing_service_from_run_name(
+        context=context, run_name=run_name
+    )
+    post_processing_service.post_process(run_name=run_name, dry_run=dry_run)
+
+
+post_process_group: click.Group
+post_process_group.add_command(post_process_sequencing_run)

cg/cli/post_process/utils.py

@@ -0,0 +1,23 @@
+from cg.exc import CgError
+from cg.models.cg_config import CGConfig
+from cg.services.run_devices.pacbio.post_processing_service import PacBioPostProcessingService
+from cg.utils.mapping import get_item_by_pattern_in_source
+
+PATTERN_TO_DEVICE_MAP: dict[str, str] = {
+    r"^r\d+_\d+_\d+/(1|2)_[^/]+$": "pacbio",
+}
+
+
+def get_post_processing_service_from_run_name(
+    context: CGConfig, run_name: str
+) -> PacBioPostProcessingService:
+    """Get the correct post-processing service based on the run name."""
+    try:
+        device: str = get_item_by_pattern_in_source(
+            source=run_name, pattern_map=PATTERN_TO_DEVICE_MAP
+        )
+    except CgError as error:
+        raise NameError(
+            f"Run name {run_name} does not match with any known sequencing run name pattern"
+        ) from error
+    return getattr(context.post_processing_services, device)

cg/constants/constants.py

@@ -195,6 +195,7 @@ class HastaSlurmPartitions(StrEnum):
 
 
 class FileExtensions(StrEnum):
+    BAM: str = ".bam"
     BED: str = ".bed"
     COMPLETE: str = ".complete"
     CONFIG: str = ".config"

cg/constants/pacbio.py

@@ -1,24 +1,30 @@
 """Constants related to PacBio sequencing."""
 
+from cg.constants import FileExtensions
+
 
 class PacBioDirsAndFiles:
-    ADAPTER_REPORT: str = "adapter.report.json"
-    BASECALLING_REPORT: str = "ccs.report.json"
+    CCS_REPORT_SUFFIX: str = "ccs_report.json"
     CONTROL_REPORT: str = "control.report.json"
     LOADING_REPORT: str = "loading.report.json"
+    HIFI_READS: str = "hifi_reads"
     RAW_DATA_REPORT: str = "raw_data.report.json"
     SMRTLINK_DATASETS_REPORT: str = "smrtlink-datasets.json"
-    CCS_REPORT_SUFFIX: str = "ccs_report.json"
+    STATISTICS_DIR: str = "statistics"
+    UNZIPPED_REPORTS_DIR: str = "unzipped_reports"
 
 
 class CCSAttributeIDs:
-    NUMBER_OF_READS: str = "ccs2.number_of_ccs_reads"
-    TOTAL_NUMBER_OF_BASES: str = "ccs2.total_number_of_ccs_bases"
-    MEAN_READ_LENGTH: str = "ccs2.mean_ccs_readlength"
-    MEDIAN_READ_LENGTH: str = "ccs2.median_ccs_readlength"
-    READ_LENGTH_N50: str = "ccs2.ccs_readlength_n50"
-    MEDIAN_ACCURACY: str = "ccs2.median_accuracy"
-    PERCENT_Q30: str = "ccs2.percent_ccs_bases_q30"
+    HIFI_READS: str = "ccs_processing.number_of_ccs_reads_q20"
+    HIFI_YIELD: str = "ccs_processing.total_number_of_ccs_bases_q20"
+    HIFI_MEAN_READ_LENGTH: str = "ccs_processing.mean_ccs_readlength_q20"
+    HIFI_MEDIAN_READ_LENGTH: str = "ccs_processing.median_ccs_readlength_q20"
+    HIFI_READ_LENGTH_N50: str = "ccs_processing.ccs_readlength_n50_q20"
+    HIFI_MEDIAN_READ_QUALITY: str = "ccs_processing.median_qv_q20"
+    PERCENT_Q30: str = "ccs_processing.base_percentage_q30"
+    FAILED_READS: str = "ccs_processing.number_of_ccs_reads_lq"
+    FAILED_YIELD: str = "ccs_processing.total_number_of_ccs_bases_lq"
+    FAILED_MEAN_READ_LENGTH: str = "ccs_processing.mean_ccs_readlength_lq"
 
 
 class ControlAttributeIDs:
@@ -48,5 +54,29 @@ class SmrtLinkDatabasesIDs:
     CELL_INDEX: str = "cellIndex"
     MOVIE_NAME: str = "metadataContextId"
     PATH: str = "path"
+    RUN_COMPLETED_AT = "createdAt"
     WELL_NAME: str = "wellName"
     WELL_SAMPLE_NAME: str = "wellSampleName"
+
+
+class PacBioHousekeeperTags:
+    CCS_REPORT: str = "ccs-report"
+    CONTROL_REPORT: str = "control-report"
+    LOADING_REPORT: str = "loading-report"
+    RAWDATA_REPORT: str = "raw-data-report"
+    DATASETS_REPORT: str = "datasets-report"
+
+
+class PacBioBundleTypes:
+    SAMPLE: str = "sample"
+    SMRT_CELL: str = "smrt_cell"
+
+
+file_pattern_to_bundle_type: dict[str, str] = {
+    PacBioDirsAndFiles.CONTROL_REPORT: PacBioBundleTypes.SMRT_CELL,
+    f".*{PacBioDirsAndFiles.CCS_REPORT_SUFFIX}$": PacBioBundleTypes.SMRT_CELL,
+    PacBioDirsAndFiles.LOADING_REPORT: PacBioBundleTypes.SMRT_CELL,
+    PacBioDirsAndFiles.RAW_DATA_REPORT: PacBioBundleTypes.SMRT_CELL,
+    PacBioDirsAndFiles.SMRTLINK_DATASETS_REPORT: PacBioBundleTypes.SMRT_CELL,
+    f"{PacBioDirsAndFiles.HIFI_READS}{FileExtensions.BAM}$": PacBioBundleTypes.SAMPLE,
+}

cg/exc.py

@@ -314,10 +314,6 @@ class OverrideCyclesError(CgError):
     """Exception raised when the override cycles are not correct."""
 
 
-class PacBioMetricsParsingError(CgError):
-    """Exception raised when PacBio metric files are not in place."""
-
-
 class Chanjo2APIClientError(CgError):
     """Exception related to the Chanjo2 API client."""
 

cg/models/cg_config.py

@@ -30,6 +30,23 @@
     FastqConcatenationService,
 )
 from cg.services.pdc_service.pdc_service import PdcService
+from cg.services.run_devices.pacbio.data_storage_service.pacbio_store_service import (
+    PacBioStoreService,
+)
+from cg.services.run_devices.pacbio.data_transfer_service.data_transfer_service import (
+    PacBioDataTransferService,
+)
+from cg.services.run_devices.pacbio.housekeeper_service.pacbio_houskeeper_service import (
+    PacBioHousekeeperService,
+)
+from cg.services.run_devices.pacbio.metrics_parser.metrics_parser import PacBioMetricsParser
+from cg.services.run_devices.pacbio.post_processing_service import PacBioPostProcessingService
+from cg.services.run_devices.pacbio.run_data_generator.pacbio_run_data_generator import (
+    PacBioRunDataGenerator,
+)
+from cg.services.run_devices.pacbio.run_file_manager.run_file_manager import (
+    PacBioRunFileManager,
+)
 from cg.services.sequencing_qc_service.sequencing_qc_service import SequencingQCService
 from cg.services.slurm_service.slurm_cli_service import SlurmCLIService
 from cg.services.slurm_service.slurm_service import SlurmService
@@ -326,6 +343,13 @@ class RunInstruments(BaseModel):
     illumina: IlluminaConfig
 
 
+class PostProcessingServices(BaseModel):
+    pacbio: PacBioPostProcessingService
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
 class CGConfig(BaseModel):
     data_input: DataInput | None = None
     database: str
@@ -382,6 +406,7 @@ class CGConfig(BaseModel):
     mutacc_auto_api_: MutaccAutoAPI = None
     pdc: CommonAppConfig | None = None
     pdc_service_: PdcService | None
+    post_processing_services_: PostProcessingServices | None = None
     pigz: CommonAppConfig | None = None
     sample_sheet_api_: IlluminaSampleSheetService | None = None
     scout: CommonAppConfig = None
@@ -425,6 +450,7 @@ class Config:
             "madeline_api_": "madeline_api",
             "mutacc_auto_api_": "mutacc_auto_api",
             "pdc_service_": "pdc_service",
+            "post_processing_services_": "post_processing_services",
             "scout_api_": "scout_api",
             "status_db_": "status_db",
             "trailblazer_api_": "trailblazer_api",
@@ -563,6 +589,38 @@ def mutacc_auto_api(self) -> MutaccAutoAPI:
             self.mutacc_auto_api_ = api
         return api
 
+    @property
+    def post_processing_services(self) -> PostProcessingServices:
+        services = self.__dict__.get("post_processing_services_")
+        if services is None:
+            LOG.debug("Instantiating post-processing services")
+            services = PostProcessingServices(
+                pacbio=self.get_pacbio_post_processing_service(),
+            )
+            self.post_processing_services_ = services
+        return services
+
+    def get_pacbio_post_processing_service(self) -> PacBioPostProcessingService:
+        LOG.debug("Instantiating PacBio post-processing service")
+        run_data_generator = PacBioRunDataGenerator()
+        file_manager = PacBioRunFileManager()
+        metrics_parser = PacBioMetricsParser(file_manager=file_manager)
+        transfer_service = PacBioDataTransferService(metrics_service=metrics_parser)
+        store_service = PacBioStoreService(
+            store=self.status_db, data_transfer_service=transfer_service
+        )
+        hk_service = PacBioHousekeeperService(
+            hk_api=self.housekeeper_api,
+            file_manager=file_manager,
+            metrics_parser=metrics_parser,
+        )
+        return PacBioPostProcessingService(
+            run_data_generator=run_data_generator,
+            hk_service=hk_service,
+            store_service=store_service,
+            sequencing_dir=self.run_instruments.pacbio.data_dir,
+        )
+
     @property
     def pdc_service(self) -> PdcService:
         service = self.__dict__.get("pdc_service_")

cg/services/illumina/post_processing/housekeeper_storage.py

@@ -78,10 +78,10 @@ def add_sample_fastq_files_to_housekeeper(
                 device_internal_id=run_directory_data.id,
                 store=store,
             ):
-                hk_api.store_fastq_path_in_housekeeper(
-                    sample_internal_id=sample_internal_id,
-                    sample_fastq_path=sample_fastq_path,
-                    flow_cell_id=run_directory_data.id,
+                hk_api.create_bundle_and_add_file_with_tags(
+                    bundle_name=sample_internal_id,
+                    file_path=sample_fastq_path,
+                    tags=[run_directory_data.id, sample_internal_id, SequencingFileTag.FASTQ],
                 )
 
 
@@ -106,10 +106,10 @@ def store_undetermined_fastq_files(
                 device_internal_id=run_directory_data.id,
                 store=store,
             ):
-                hk_api.store_fastq_path_in_housekeeper(
-                    sample_internal_id=sample_id,
-                    sample_fastq_path=fastq_path,
-                    flow_cell_id=run_directory_data.id,
+                hk_api.create_bundle_and_add_file_with_tags(
+                    bundle_name=sample_id,
+                    file_path=fastq_path,
+                    tags=[run_directory_data.id, SequencingFileTag.FASTQ, sample_id],
                 )