Add pipeline level ci

.github/workflows/ci_nextflow.yml

@@ -0,0 +1,60 @@
+name: pipeline CI
+# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  release:
+    types: [published]
+
+env:
+  NXF_ANSI_LOG: false
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Run pipeline with test data
+    # Only run on push if this is the nf-core dev branch (merged PRs)
+    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'mathysgrapotte/stimulus') }}"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        NXF_VER:
+          - "23.04.0"
+          - "latest-everything"
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: "${{ matrix.NXF_VER }}"
+
+      - name: Disk space cleanup
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+
+      - name: Run pipeline with test data
+        # TODO: Get rid of loca profile when the nf-core like configs are in
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile test,local
+
+  test_stub:
+    name: Stub run of the pipeline
+    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'mathysgrapotte/stimulus') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+      - name: Run pipeline with stub-run in alphafold2 split mode
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile test_stub

bin/src/learner/raytune_learner.py

@@ -16,6 +16,7 @@ def __init__(self, config_path: str, model_class: nn.Module, data_path: str, exp
         """
         Initialize the TuneWrapper with the paths to the config, model, and data.
         """
+        self.best_config = None
         self.config = YamlRayConfigLoader(config_path).get_config()
         self.config["model"] = model_class
         self.config["experiment"] = experiment_object
@@ -50,6 +51,13 @@ def tune(self) -> None:
         """
         return self.tuner.fit() 
 
+    def store_best_config(self, path: str) -> None:
+        """
+        Store the best config in a file.
+        """
+        with open(path, "w") as f:
+            f.write(str(self.best_config))  
+
 class TuneModel(Trainable):
 
     def setup(self, config: dict) -> None:

configs/modules.config

@@ -10,24 +10,23 @@
 
 process {
 
-
-    withName: "GENERATE_FASTA|GENERATE_FROM_FASTA" {
-        ext.args = { [ params.dna_seq_len          ? "-sl ${params.dna_seq_len}"         : '-sl 100',
-                       params.motif_tag            ? "-t ${params.motif_tag}"            : '-t 5',
-                       params.non_motif_tag        ? "-u ${params.non_motif_tag}"        : '-u 0',
-                       params.num_seq              ? "-ns ${params.num_seq}"             : '',
-                       params.motif_start          ? "-p ${params.motif_start}"          : ''
-                       ].flatten().unique(false).join(' ').trim()
-        }
-        ext.prefix = { params.generated_fasta ? params.generated_fasta : null }
+    // withName: "GENERATE_FASTA|GENERATE_FROM_FASTA" {
+    //     ext.args = { [ params.dna_seq_len          ? "-sl ${params.dna_seq_len}"         : '-sl 100',
+    //                    params.motif_tag            ? "-t ${params.motif_tag}"            : '-t 5',
+    //                    params.non_motif_tag        ? "-u ${params.non_motif_tag}"        : '-u 0',
+    //                    params.num_seq              ? "-ns ${params.num_seq}"             : '',
+    //                    params.motif_start          ? "-p ${params.motif_start}"          : ''
+    //                    ].flatten().unique(false).join(' ').trim()
+    //     }
+    //     ext.prefix = { params.generated_fasta ? params.generated_fasta : null }
 
-        // the outdir has to be the one the user specify plus stuff that makes it run unique
-        publishDir = [
-            path: { "${params.outdir}/${workflow.runName}_" + "${workflow.start}".replaceAll('[-:]', '_').split('\\.')[0] },
-            mode: params.publish_dir_mode,
-            overwrite: true
-        ]
-    }
+    //     // the outdir has to be the one the user specify plus stuff that makes it run unique
+    //     publishDir = [
+    //         path: { "${params.outdir}/${workflow.runName}_" + "${workflow.start}".replaceAll('[-:]', '_').split('\\.')[0] },
+    //         mode: params.publish_dir_mode,
+    //         overwrite: true
+    //     ]
+    // }
 
     withName: "TORCH_TUNE" {
 

configs/test.config

@@ -10,6 +10,6 @@ params {
     csv         = "${projectDir}/bin/tests/test_data/dna_experiment/test.csv"
     exp_conf    = "${projectDir}/examples/test.json"
     model       = "${projectDir}/bin/tests/test_model/dnatofloatmodel.py"
-    train_conf  = "${projectDir}/bin/tests/test_model/simple.config"
+    train_conf  = "${projectDir}/bin/tests/test_model/simple_config.yaml"
 
 }

configs/test_stub.config

@@ -16,7 +16,7 @@ params {
     csv         = "${projectDir}/bin/tests/test_data/dna_experiment/test.csv"
     exp_conf    = "${projectDir}/examples/test.json"
     model       = "${projectDir}/bin/tests/test_model/dnatofloatmodel.py"
-    train_conf  = "${projectDir}/bin/tests/test_model/simple.config"
+    train_conf  = "${projectDir}/bin/tests/test_model/simple_config.yaml"
 }
 
 process {

nextflow_schema.json

@@ -37,7 +37,7 @@
           "format": "directory-path",
           "description": "Training config",
           "fa_icon": "fas fa-folder-open",
-          "default": "${projectDir}/bin/tests/test_model/simple.config"
+          "default": "${projectDir}/bin/tests/test_model/simple_config.yaml"
         }
       }
     }