From f86cb4e28727a61c803384dedb8d3b33bf993a0e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 28 Apr 2023 16:13:15 +0200 Subject: [PATCH 001/249] bump version to 2.9dev --- .gitpod.yml | 2 +- CHANGELOG.md | 12 ++++++++++++ setup.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 263fcc41db..a93e660516 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 8921d75fea..7e5d011907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # nf-core/tools: Changelog +# v2.9dev + +### Template + +### Linting + +### Modules + +### Subworkflows + +### General + # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27] ### Template diff --git a/setup.py b/setup.py index 9b0d9fb6af..4438850720 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.8" +version = "2.9dev" with open("README.md") as f: readme = f.read() From d998e788e7ab1953b07a98918b9316c13dae6c6c Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Sat, 29 Apr 2023 17:52:49 +0200 Subject: [PATCH 002/249] Typo tweaks and minor clarifications --- nf_core/pipeline-template/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index e2ca15a8e6..5ac65123ed 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -82,11 +82,11 @@ nextflow run {{ name }} \ {% if branded -%} -For more details, please refer to the [usage documentation](https://nf-co.re/{{ short_name }}/usage) and the [parameter documentation](https://nf-co.re/{{ short_name }}/parameters). +For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/{{ short_name }}/usage) and the [parameter documentation](https://nf-co.re/{{ short_name }}/parameters). ## Pipeline output -To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/{{ short_name }}/results) tab on the nf-core website pipeline page. +To see the results of an example test run with a full size dataset refer to the [results](https://nf-co.re/{{ short_name }}/results) tab on the nf-core website pipeline page. For more details about the output files and reports, please refer to the [output documentation](https://nf-co.re/{{ short_name }}/output). From 6795985b14d8496789dcfed50d20a102d142e89e Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 2 May 2023 09:55:22 +0100 Subject: [PATCH 003/249] Container warns if permission is denied --- nf_core/modules/lint/main_nf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index e9e18b3e12..b59e166441 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -328,8 +328,11 @@ def check_process_section(self, lines, fix_version, progress_bar): log.debug(f"Unable to connect to url '{urlunparse(url)}' due to error: {e}") self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) continue - if response.status_code != 200: - self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) + if not response.ok: + if 400 <= response.status_code < 500: + self.warned.append(("container_links", "Access to container URL denied", self.main_nf)) + else: + self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) # Check that all bioconda packages have build numbers # Also check for newer versions From ec334da8bbd13b81e601df48fb8d9a44dc203528 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 2 May 2023 09:58:40 +0100 Subject: [PATCH 004/249] Change to specific status codes --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index b59e166441..dc29513dc2 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -329,7 +329,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) continue if not response.ok: - if 400 <= response.status_code < 500: + if response.status_code in [400, 401, 402, 403]: self.warned.append(("container_links", "Access to container URL denied", self.main_nf)) else: self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) From 8cb66f5e7e5b6f4521ec2d27f0d98763bf67a1fa Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 2 May 2023 10:01:47 +0100 Subject: [PATCH 005/249] CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0db70fce12..13e340465d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Added config `docker.registry` to pipeline template for a configurable default container registry when using Docker containers. Defaults to `quay.io` ([#2133](https://github.com/nf-core/tools/pull/2133)) - Add tower.yml file to the pipeline template ([#2251](https://github.com/nf-core/tools/pull/2251)) - Add mastodon badge to README ([#2253](https://github.com/nf-core/tools/pull/2253)) +- Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270)) ### Linting From 7d87d6d389fd3cad951a673c1194d1fa73f8fc40 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 2 May 2023 11:01:10 +0100 Subject: [PATCH 006/249] Raise warning for modules --- nf_core/modules/lint/main_nf.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 58d27f0431..3a86e0f314 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -332,10 +332,13 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) continue if not response.ok: - if response.status_code in [400, 401, 402, 403]: - self.warned.append(("container_links", "Access to container URL denied", self.main_nf)) - else: - self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) + self.warned.append( + ( + "container_links", + f"Access to container {response.url} denied, status code: {response.status_code}", + self.main_nf, + ) + ) # Check that all bioconda packages have build numbers # Also check for newer versions From e4cd14a0fccaada8596c4f2437b210080b179a12 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 2 May 2023 11:48:27 +0100 Subject: [PATCH 007/249] More explicit errors regarding connecting to container URLs --- nf_core/modules/lint/main_nf.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 3a86e0f314..5c33d034c3 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -332,13 +332,23 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) continue if not response.ok: - self.warned.append( - ( - "container_links", - f"Access to container {response.url} denied, status code: {response.status_code}", - self.main_nf, + if response.status_code in [400, 401, 402, 403]: + self.warned.append( + ( + "container_links", + f"Access to container {response.url} denied, status code: {response.status_code}", + self.main_nf, + ) + ) + + else: + self.failed.append( + ( + "container_links", + f"Unable to connect to {response.url}, status code: {response.status_code}", + self.main_nf, + ) ) - ) # Check that all bioconda packages have build numbers # Also check for newer versions From 1087e8f158c90e34371c31ca8b43cfff2697a28c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 2 May 2023 15:47:01 +0200 Subject: [PATCH 008/249] only check for url symbols for docker containers url --- nf_core/modules/lint/main_nf.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 5c33d034c3..703dba18a0 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -597,9 +597,5 @@ def _container_type(line): if url_match: return "singularity" return None - if ( - line.startswith("biocontainers/") - or line.startswith("quay.io/") - or (line.count("/") == 1 and line.count(":") == 1) - ): + if line.count("/") >= 1 and line.count(":") == 1: return "docker" From f067e871ea0dec3a3a6c719e85dc0e3d89fd3245 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 2 May 2023 14:48:11 +0100 Subject: [PATCH 009/249] Better error message when unable to connect to container --- nf_core/modules/lint/main_nf.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 703dba18a0..102d5ead12 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -332,23 +332,13 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) continue if not response.ok: - if response.status_code in [400, 401, 402, 403]: - self.warned.append( - ( - "container_links", - f"Access to container {response.url} denied, status code: {response.status_code}", - self.main_nf, - ) - ) - - else: - self.failed.append( - ( - "container_links", - f"Unable to connect to {response.url}, status code: {response.status_code}", - self.main_nf, - ) + self.failed.append( + ( + "container_links", + f"Unable to connect to {response.url}, status code: {response.status_code}", + self.main_nf, ) + ) # Check that all bioconda packages have build numbers # Also check for newer versions From 3a224754ed07ed3b6d8014440460ffe5ff91d23b Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Wed, 3 May 2023 13:04:17 -0400 Subject: [PATCH 010/249] modified email function so that it does not require multiqc_report --- .../lib/NfcoreTemplate.groovy | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 2cb8b41388..74a340978d 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -94,19 +94,21 @@ class NfcoreTemplate { // On success try attach the multiqc report def mqc_report = null - try { - if (workflow.success) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" + if (multiqc_report) { + try { + if (workflow.success) { + mqc_report = multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { + if (mqc_report.size() > 1) { + log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" + } + mqc_report = mqc_report[0] } - mqc_report = mqc_report[0] } - } - } catch (all) { - if (multiqc_report) { - log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + } catch (all) { + if (multiqc_report) { + log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + } } } @@ -128,8 +130,12 @@ class NfcoreTemplate { def email_html = html_template.toString() // Render the sendmail template - def max_multiqc_email_size = params.max_multiqc_email_size as nextflow.util.MemoryUnit - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] + def max_multiqc_email_size = params.containsKey("max_multiqc_email_size") ? params.max_multiqc_email_size as nextflow.util.MemoryUnit : 0 + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile : null, mqcMaxSize : null ] + if (mqc_report) { + smail_fields["mqcFile"] = mqc_report + smail_fields["mqcMaxSize"] = max_multiqc_email_size.toBytes() + } def sf = new File("$projectDir/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) def sendmail_html = sendmail_template.toString() @@ -145,8 +151,10 @@ class NfcoreTemplate { } catch (all) { // Catch failures and try with plaintext def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { - mail_cmd += [ '-A', mqc_report ] + if (mqc_report) { + if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { + mail_cmd += [ '-A', mqc_report ] + } } mail_cmd.execute() << email_html log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" From e7afe5ce0bd6aa845c5c434bbefa5b8b5270435f Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Wed, 3 May 2023 13:12:48 -0400 Subject: [PATCH 011/249] updated change log with email parameter change --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8921d75fea..94504e8596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # nf-core/tools: Changelog +# v2.9dev + +### Template + +- The `email` method no longer requires a `multiqc_report` parameter + # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27] ### Template From 79743a7822467f1e854e39bca7e68335f25a87ad Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Wed, 3 May 2023 13:13:16 -0400 Subject: [PATCH 012/249] linting --- nf_core/pipeline-template/lib/NfcoreTemplate.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 74a340978d..8590f0d35a 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -130,11 +130,11 @@ class NfcoreTemplate { def email_html = html_template.toString() // Render the sendmail template - def max_multiqc_email_size = params.containsKey("max_multiqc_email_size") ? params.max_multiqc_email_size as nextflow.util.MemoryUnit : 0 + def max_multiqc_email_size = params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size as nextflow.util.MemoryUnit : 0 def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile : null, mqcMaxSize : null ] if (mqc_report) { - smail_fields["mqcFile"] = mqc_report - smail_fields["mqcMaxSize"] = max_multiqc_email_size.toBytes() + smail_fields['mqcFile'] = mqc_report + smail_fields['mqcMaxSize'] = max_multiqc_email_size.toBytes() } def sf = new File("$projectDir/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) From c94b08bf4deed7d46af9592286a390c47a2e4c1d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 4 May 2023 07:17:28 +0200 Subject: [PATCH 013/249] GitPod base image: Always self-update to the latest version of Nextflow. --- CHANGELOG.md | 2 ++ nf_core/gitpod/gitpod.Dockerfile | 15 +++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3c0792519..f0940d53b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ ### General +- GitPod base image: Always self-update to the latest version of Nextflow. + # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27] ### Template diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 417208a20d..252b9bd088 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -38,13 +38,16 @@ RUN conda config --add channels defaults && \ conda config --set channel_priority strict && \ conda install --quiet --yes --name base mamba && \ mamba install --quiet --yes --name base \ - nextflow=22.10.1 \ - nf-core \ - nf-test \ - black \ - prettier \ - pytest-workflow && \ + nextflow \ + nf-core \ + nf-test \ + black \ + prettier \ + pytest-workflow && \ mamba clean --all -f -y +# Update Nextflow +RUN nextflow self-update + # Install nf-core RUN python -m pip install . From 5cd0b867fa06b762762b97530c79aa5915a57a37 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 4 May 2023 07:23:29 +0200 Subject: [PATCH 014/249] GitPod config: Update Nextflow in init. Install pre-commit. --- .gitpod.yml | 1 + CHANGELOG.md | 3 ++- nf_core/gitpod/gitpod.Dockerfile | 1 + nf_core/pipeline-template/.gitpod.yml | 5 +++++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index a93e660516..1cc63b197f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -5,6 +5,7 @@ tasks: python -m pip install -e . python -m pip install -r requirements-dev.txt pre-commit install --install-hooks + nextflow self-update vscode: extensions: # based on nf-core.nf-core-extensionpack - codezombiech.gitignore # Language support for .gitignore files diff --git a/CHANGELOG.md b/CHANGELOG.md index f0940d53b9..11bc0e1a69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ ### General -- GitPod base image: Always self-update to the latest version of Nextflow. +- GitPod base image: Always self-update to the latest version of Nextflow. Add [pre-commit](https://pre-commit.com/) dependency. +- GitPod configs: Update Nextflow as an init task, init pre-commit in pipeline config. # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27] diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 252b9bd088..5fce2f055b 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -43,6 +43,7 @@ RUN conda config --add channels defaults && \ nf-test \ black \ prettier \ + pre-commit \ pytest-workflow && \ mamba clean --all -f -y diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index 85d95ecc8e..25488dcc08 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -1,4 +1,9 @@ image: nfcore/gitpod:latest +tasks: + - name: Update Nextflow and setup pre-commit + command: | + pre-commit install --install-hooks + nextflow self-update vscode: extensions: # based on nf-core.nf-core-extensionpack From f2d0eca7f120cf556d8481697c1d6284c6209961 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 4 May 2023 10:05:16 +0200 Subject: [PATCH 015/249] Add more info about reference genome As we always display params used for that, because default is the map inside the genomes map --- nf_core/pipeline-template/lib/NfcoreSchema.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index 9b34804d6d..8d88874fe8 100755 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -327,6 +327,7 @@ class NfcoreSchema { } } output += "!! Only displaying parameters that differ from the pipeline defaults !!\n" + output += "!! Displaying all reference genome parameters !!\n" output += NfcoreTemplate.dashedLine(params.monochrome_logs) return output } From 575e25bae370e1f789bd7996738491413e8c9b82 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 4 May 2023 11:14:58 +0200 Subject: [PATCH 016/249] Update nf_core/pipeline-template/lib/NfcoreSchema.groovy --- nf_core/pipeline-template/lib/NfcoreSchema.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index 8d88874fe8..02d6641f33 100755 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -327,7 +327,7 @@ class NfcoreSchema { } } output += "!! Only displaying parameters that differ from the pipeline defaults !!\n" - output += "!! Displaying all reference genome parameters !!\n" + output += "Displaying all reference genome parameters\n" output += NfcoreTemplate.dashedLine(params.monochrome_logs) return output } From affb710692d5eeb056d2477cd60fe197259c898b Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 4 May 2023 16:08:55 +0200 Subject: [PATCH 017/249] Add stub to the modules template --- nf_core/module-template/modules/main.nf | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 83cdf90b92..404d38094d 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -94,4 +94,26 @@ process {{ component_name_underscore|upper }} { {{ tool }}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' )) END_VERSIONS """ + + stub: + def args = task.ext.args ?: '' + {% if has_meta -%} + def prefix = task.ext.prefix ?: "${meta.id}" + {%- endif %} + {% if not_empty_template -%} + // TODO nf-core: A stub section should mimic the execution of the original module as best as possible + // Have a look at the following examples: + // Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63 + // Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54 + {%- endif %} + """ + {% if not_empty_template -%} + touch ${prefix}.bam + {%- endif %} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + {{ tool }}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' )) + END_VERSIONS + """ } From 4bad234df707bf886d172f97362ff4c2daa2a9c7 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 4 May 2023 16:34:13 +0100 Subject: [PATCH 018/249] error if quay.io found in docker container name Changes: - Linting error if container starts with quay.io/biocontainers but not quay.io/biocontainers/mulled Fixes #2276 --- CHANGELOG.md | 1 + nf_core/modules/lint/main_nf.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3c0792519..d64c03267e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Linting - Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270)) +- Error if module container specification has quay.io as prefix when it shouldn't have. ### Modules diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 102d5ead12..bfdb92c5cf 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -290,6 +290,14 @@ def check_process_section(self, lines, fix_version, progress_bar): else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = None + if l.startswith("quay.io/biocontainers") and not l.startswith("quay.io/biocontainers/mulled"): + self.failed.append( + ( + "container_links", + "quay.io/biocontainers prefix found, please use biocontainers instead", + self.main_nf, + ) + ) if l.startswith("biocontainers/"): # When we think it is a biocontainer, assume we are querying quay.io/biocontainers and insert quay.io as prefix l = "quay.io/" + l From 5e054c260228e25f0848da30af0491b4b90e073d Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 4 May 2023 21:28:56 +0100 Subject: [PATCH 019/249] Remove cleanup=true from test_full.config --- CHANGELOG.md | 3 +++ nf_core/pipeline-template/conf/test_full.config | 2 -- nf_core/pipeline-template/docs/usage.md | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3c0792519..fb5e81e6e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### Template +- Remove `cleanup = true` from `test_full.config` in pipeline template +- Fix usage docs for specifying `params.yaml` + ### Linting - Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270)) diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index 46b165a910..d92692fa94 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -10,8 +10,6 @@ ---------------------------------------------------------------------------------------- */ -cleanup = true - params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 73e1132541..e89c2a7332 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -61,7 +61,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p The typical command for running the pipeline is as follows: ```bash -nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile docker +nextflow run {{ name }} --input ./samplesheet.csv --outdir ./results --genome GRCh37 -profile docker ``` This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. @@ -80,7 +80,8 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. > ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args). -> The above pipeline run specified with a params file in yaml format: + +The above pipeline run specified with a params file in yaml format: ```bash nextflow run {{ name }} -profile docker -params-file params.yaml @@ -92,7 +93,6 @@ with `params.yaml` containing: input: './samplesheet.csv' outdir: './results/' genome: 'GRCh37' -input: 'data' <...> ``` From aea396a68e64dde41b1e00ce70e8da522bac828a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 5 May 2023 08:49:41 +0200 Subject: [PATCH 020/249] More verbose error explanation --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index bfdb92c5cf..83ce4502ec 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -294,7 +294,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append( ( "container_links", - "quay.io/biocontainers prefix found, please use biocontainers instead", + "'quay.io/biocontainers/' container prefix found, please use just 'biocontainers/' instead", self.main_nf, ) ) From 232b11de33b8cb93a77b85e0439d7e236a5f8206 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 5 May 2023 09:00:07 +0200 Subject: [PATCH 021/249] updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3c0792519..4a62d70aa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Added stub in modules template ([#2277])(https://github.com/nf-core/tools/pull/2277) + ### Linting - Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270)) From 86aee0caab95be421af718f96f9b85e74bc6f096 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 5 May 2023 10:45:09 +0100 Subject: [PATCH 022/249] Add passed test if quay.io is correct --- nf_core/modules/lint/main_nf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 83ce4502ec..0af8e6357a 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -298,6 +298,8 @@ def check_process_section(self, lines, fix_version, progress_bar): self.main_nf, ) ) + else: + self.passed.append(("container_links", "Container prefix is correct", self.main_nf)) if l.startswith("biocontainers/"): # When we think it is a biocontainer, assume we are querying quay.io/biocontainers and insert quay.io as prefix l = "quay.io/" + l From 4351a34af621b087832c17b7d020197619e6e7f8 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 5 May 2023 10:47:39 +0100 Subject: [PATCH 023/249] Container status check only occurs once Puts container status check into a single if statement --- nf_core/modules/lint/main_nf.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0af8e6357a..96d03421eb 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -271,15 +271,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf)) singularity_tag = None url = urlparse(l.split("'")[0]) - # lint double quotes - if l.count('"') > 2: - self.failed.append( - ( - "container_links", - "Too many double quotes found when specifying singularity container", - self.main_nf, - ) - ) + if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 @@ -304,17 +296,26 @@ def check_process_section(self, lines, fix_version, progress_bar): # When we think it is a biocontainer, assume we are querying quay.io/biocontainers and insert quay.io as prefix l = "quay.io/" + l url = urlparse(l.split("'")[0]) - # lint double quotes - if l.count('"') > 2: - self.failed.append( - ("container_links", "Too many double quotes found when specifying docker container", self.main_nf) - ) + # lint double quotes - if l.startswith("container"): + if l.startswith("container") or _container_type(l) == "docker" or _container_type(l) == "singularity": if l.count('"') > 2: self.failed.append( - ("container_links", "Too many double quotes found when specifying containers", self.main_nf) + ( + "container_links", + f"Too many double quotes found when specifying container: {l.lstrip('container ')}", + self.main_nf, + ) + ) + else: + self.passed.append( + ( + "container_links", + f"Correct number of double quotes found when specifying container: {l.lstrip('container ')}", + self.main_nf, + ) ) + # lint more than one container in the same line if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l): self.warned.append( From 1f9caf5995ea68282c8f9fc2eb80fb26c7613dac Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 5 May 2023 13:23:00 +0100 Subject: [PATCH 024/249] Fix: Detect docker name and append quay.io Changes: - Detects simple docker name by having a single slash and colon in the string - adds quay.io as default registry for this docker container - Should fix problems where nfcore/ubuntu:20.04 did not find correct URL fixes #2280 --- CHANGELOG.md | 1 + nf_core/modules/lint/main_nf.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 116e55eb30..42d3f10ed6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Linting - Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270)) +- Detect if container is 'simple name' and try to contact quay.io server by default. ### Modules diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 102d5ead12..90f92d0ff7 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -290,8 +290,9 @@ def check_process_section(self, lines, fix_version, progress_bar): else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = None - if l.startswith("biocontainers/"): - # When we think it is a biocontainer, assume we are querying quay.io/biocontainers and insert quay.io as prefix + # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04) + # If so, add quay.io as default container prefix + if l.count("/") == 1 and l.count(":") == 1: l = "quay.io/" + l url = urlparse(l.split("'")[0]) # lint double quotes From 09817321b2273f7af359fad544f87c3dd851ef40 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 5 May 2023 13:26:52 +0100 Subject: [PATCH 025/249] CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42d3f10ed6..4f90fb86c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ ### Linting - Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270)) -- Detect if container is 'simple name' and try to contact quay.io server by default. +- Detect if container is 'simple name' and try to contact quay.io server by default ([#2281](https://github.com/nf-core/tools/pull/2281)) ### Modules From c9093fe31dc0ba4224fbb078b29c42d2fec80a64 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Fri, 5 May 2023 13:09:32 -0400 Subject: [PATCH 026/249] simpler implementation --- .../lib/NfcoreTemplate.groovy | 38 ++++++++----------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 8590f0d35a..2cb8b41388 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -94,22 +94,20 @@ class NfcoreTemplate { // On success try attach the multiqc report def mqc_report = null - if (multiqc_report) { - try { - if (workflow.success) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" - } - mqc_report = mqc_report[0] + try { + if (workflow.success) { + mqc_report = multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { + if (mqc_report.size() > 1) { + log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" } - } - } catch (all) { - if (multiqc_report) { - log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + mqc_report = mqc_report[0] } } + } catch (all) { + if (multiqc_report) { + log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + } } // Check if we are only sending emails on failure @@ -130,12 +128,8 @@ class NfcoreTemplate { def email_html = html_template.toString() // Render the sendmail template - def max_multiqc_email_size = params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size as nextflow.util.MemoryUnit : 0 - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile : null, mqcMaxSize : null ] - if (mqc_report) { - smail_fields['mqcFile'] = mqc_report - smail_fields['mqcMaxSize'] = max_multiqc_email_size.toBytes() - } + def max_multiqc_email_size = params.max_multiqc_email_size as nextflow.util.MemoryUnit + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] def sf = new File("$projectDir/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) def sendmail_html = sendmail_template.toString() @@ -151,10 +145,8 @@ class NfcoreTemplate { } catch (all) { // Catch failures and try with plaintext def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if (mqc_report) { - if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { - mail_cmd += [ '-A', mqc_report ] - } + if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { + mail_cmd += [ '-A', mqc_report ] } mail_cmd.execute() << email_html log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" From d5d1887b5167e94babe95122739f7033c607828c Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Fri, 5 May 2023 13:14:31 -0400 Subject: [PATCH 027/249] fixed issue with max_multiqc_email_size --- nf_core/pipeline-template/lib/NfcoreTemplate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 2cb8b41388..a1a726d69f 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -128,7 +128,7 @@ class NfcoreTemplate { def email_html = html_template.toString() // Render the sendmail template - def max_multiqc_email_size = params.max_multiqc_email_size as nextflow.util.MemoryUnit + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] def sf = new File("$projectDir/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) From eae38cb93fe1f4ffff3674eaad65d16489de51f8 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Fri, 5 May 2023 15:09:30 -0400 Subject: [PATCH 028/249] updated CHANGELOG with new fix --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b54d5f1365..f202c52764 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Template -- The `email` method no longer requires a `multiqc_report` parameter +- `params.max_multiqc_email_size` is no longer required - Remove `cleanup = true` from `test_full.config` in pipeline template - Fix usage docs for specifying `params.yaml` - Added stub in modules template ([#2277])(https://github.com/nf-core/tools/pull/2277) [Contributed by @nvnieuwk] From 55f5200a0330b5e504997e8d0863e94403def114 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Mon, 8 May 2023 12:07:56 +0200 Subject: [PATCH 029/249] Fix indent --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 4ef0fcd5e8..ec9300ef4f 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -84,7 +84,7 @@ profiles { debug { dumpHashes = true process.beforeScript = 'echo $HOSTNAME' - cleanup = false + cleanup = false } conda { conda.enabled = true From 46c1dea607937dfe710da12e9dd35809f88ff682 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 9 May 2023 13:32:32 +0100 Subject: [PATCH 030/249] Checks for any instance of quay.io in container name --- nf_core/modules/lint/main_nf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 6530549efa..4f630bbb23 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -282,16 +282,16 @@ def check_process_section(self, lines, fix_version, progress_bar): else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = NoneD - if l.startswith("quay.io/biocontainers") and not l.startswith("quay.io/biocontainers/mulled"): + if l.startswith("quay.io/"): self.failed.append( ( "container_links", - "'quay.io/biocontainers/' container prefix found, please use just 'biocontainers/' instead", + f"'quay.io//:' container name found, please use just '/:' instead. Offending container name: {l}", self.main_nf, ) ) else: - self.passed.append(("container_links", "Container prefix is correct", self.main_nf)) + self.passed.append(("container_links", f"Container prefix is correct", self.main_nf)) # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04) # If so, add quay.io as default container prefix From c786f2ac6bca26221e3e8bd6dc8825366fd09e6a Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 9 May 2023 14:46:55 +0100 Subject: [PATCH 031/249] Improved log message --- nf_core/modules/lint/main_nf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 4f630bbb23..8150e7e839 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -283,10 +283,11 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = NoneD if l.startswith("quay.io/"): + l_stripped = re.sub("\W+$", "", l) self.failed.append( ( "container_links", - f"'quay.io//:' container name found, please use just '/:' instead. Offending container name: {l}", + f"{l_stripped} container name found, please use just 'organisation/container:tag' instead.", self.main_nf, ) ) From 00bd8efae41f63bb68a6d4b51f6beaaea0ab7878 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 9 May 2023 16:15:04 +0200 Subject: [PATCH 032/249] update gitlab sha --- tests/modules/patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 95cc2cad95..338d890f2f 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -18,7 +18,7 @@ """ ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "0245a9277d51a47c8aa68d264d294cf45312fab8" +CORRECT_SHA = "1dff30bfca2d98eb7ac7b09269a15e822451d99f" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align" From 3e1a44a96fdd730c59ffc9d68485f279e2d0f037 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 11 May 2023 09:21:44 +0100 Subject: [PATCH 033/249] Move docker.registry definition out of docker profile --- nf_core/pipeline-template/nextflow.config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ec9300ef4f..fdd9bce5f1 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -107,7 +107,6 @@ profiles { } docker { docker.enabled = true - docker.registry = 'quay.io' docker.userEmulation = true conda.enabled = false singularity.enabled = false @@ -198,6 +197,9 @@ env { // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] +// Set default docker registry (will not be used unless pulling docker images) +docker.registry = 'quay.io' + def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true From bd83d3d872cb4680a1492c5264a8e81bfae3472c Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 11 May 2023 09:32:33 +0100 Subject: [PATCH 034/249] Move podman.registry out of profile scope too --- nf_core/pipeline-template/nextflow.config | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index fdd9bce5f1..2650572c93 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -130,7 +130,6 @@ profiles { } podman { podman.enabled = true - podman.registry = 'quay.io' conda.enabled = false docker.enabled = false singularity.enabled = false @@ -174,6 +173,12 @@ profiles { test_full { includeConfig 'conf/test_full.config' } } +// Set default registry for Docker and Podman independent of -profile +// Will not be used unless Docker / Podman are enabled +// Set to your registry if you have a mirror of containers +docker.registry = 'quay.io' +podman.registry = 'quay.io' + {% if igenomes %} // Load igenomes.config if required if (!params.igenomes_ignore) { @@ -197,9 +202,6 @@ env { // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] -// Set default docker registry (will not be used unless pulling docker images) -docker.registry = 'quay.io' - def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true From 62c1db44a4f8437e26ad2a4c37941720e5e6df34 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 11 May 2023 18:08:36 +0200 Subject: [PATCH 035/249] remove aws_tower profile --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 4942167b12..9511ea57be 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -27,7 +27,7 @@ jobs: { "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" } - profiles: test_full,aws_tower + profiles: test_full - uses: actions/upload-artifact@v3 with: name: Tower debug log file From 14d4b2e0ccc0575fceaea4f93ef1317c3804d3a2 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 11 May 2023 18:15:21 +0200 Subject: [PATCH 036/249] Update awstest.yml --- nf_core/pipeline-template/.github/workflows/awstest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 7f80cf1bb5..6e1fd8c92a 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -22,7 +22,7 @@ jobs: { "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-test-${{ github.sha }}{% endraw %}" } - profiles: test,aws_tower + profiles: test - uses: actions/upload-artifact@v3 with: name: Tower debug log file From 8fe2e0dbbfa0705aee6067a9d7757e54882a5c40 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 12 May 2023 10:01:56 +0200 Subject: [PATCH 037/249] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b094020e65..945b56f15d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - Remove `cleanup = true` from `test_full.config` in pipeline template - Fix usage docs for specifying `params.yaml` - Added stub in modules template ([#2277])(https://github.com/nf-core/tools/pull/2277) [Contributed by @nvnieuwk] +- Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286) +- Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) ### Linting From 94dcf28081a3d4474715ee534237bd87b40b8aa3 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Fri, 12 May 2023 11:35:03 -0400 Subject: [PATCH 038/249] remove-tracedir-param --- CHANGELOG.md | 1 + nf_core/pipeline-template/nextflow.config | 9 ++++----- nf_core/pipeline-template/nextflow_schema.json | 7 ------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 945b56f15d..936de5b124 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Added stub in modules template ([#2277])(https://github.com/nf-core/tools/pull/2277) [Contributed by @nvnieuwk] - Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286) - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) +- Remove `--tracedir` parameter ### Linting diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2650572c93..8ce5d207e9 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -29,7 +29,6 @@ params { // Boilerplate options outdir = null - tracedir = "${params.outdir}/pipeline_info" publish_dir_mode = 'copy' email = null email_on_fail = null @@ -205,19 +204,19 @@ process.shell = ['/bin/bash', '-euo', 'pipefail'] def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true - file = "${params.tracedir}/execution_timeline_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/execution_timeline_${trace_timestamp}.html" } report { enabled = true - file = "${params.tracedir}/execution_report_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html" } trace { enabled = true - file = "${params.tracedir}/execution_trace_${trace_timestamp}.txt" + file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" } dag { enabled = true - file = "${params.tracedir}/pipeline_dag_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/pipeline_dag_${trace_timestamp}.html" } manifest { diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 2743562d6c..ee7ac1932c 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -243,13 +243,6 @@ "description": "Custom MultiQC yaml file containing HTML including a methods description.", "fa_icon": "fas fa-cog" }, - "tracedir": { - "type": "string", - "description": "Directory to keep pipeline Nextflow logs and reports.", - "default": "${params.outdir}/pipeline_info", - "fa_icon": "fas fa-cogs", - "hidden": true - }, "validate_params": { "type": "boolean", "description": "Boolean whether to validate parameters against the schema at runtime", From 22fd8b0819bd5ce210407774b0a87c49d6a7aeff Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 15 May 2023 09:48:43 +0100 Subject: [PATCH 039/249] bugfix: the actual pipeline name should be shown --- nf_core/pipeline-template/assets/slackreport.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/assets/slackreport.json b/nf_core/pipeline-template/assets/slackreport.json index 043d02f275..ec03b3968a 100644 --- a/nf_core/pipeline-template/assets/slackreport.json +++ b/nf_core/pipeline-template/assets/slackreport.json @@ -3,7 +3,7 @@ { "fallback": "Plain-text summary of the attachment.", "color": "<% if (success) { %>good<% } else { %>danger<%} %>", - "author_name": "sanger-tol/readmapping v${version} - ${runName}", + "author_name": "{{ name }} v${version} - ${runName}", "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", "fields": [ From 6a95bf30323a65528fa11a9952d71b299bba6c9d Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 15 May 2023 09:50:51 +0100 Subject: [PATCH 040/249] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8921d75fea..6b8d681cac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # nf-core/tools: Changelog +# v2.9 (TBC) + +### Template + +- Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) + # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27] ### Template From ec4ccad19f5977817667a933ea3e06b478ea2f23 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Tue, 16 May 2023 11:16:04 +0200 Subject: [PATCH 041/249] Update nf_core/pipeline-template/lib/NfcoreSchema.groovy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/pipeline-template/lib/NfcoreSchema.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index 02d6641f33..707f58a276 100755 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -327,7 +327,9 @@ class NfcoreSchema { } } output += "!! Only displaying parameters that differ from the pipeline defaults !!\n" + {% if igenomes -%} output += "Displaying all reference genome parameters\n" + {% endif -%} output += NfcoreTemplate.dashedLine(params.monochrome_logs) return output } From 06b34de920fe0e09875482a19fae260954c02347 Mon Sep 17 00:00:00 2001 From: SusiJo Date: Thu, 18 May 2023 14:57:30 +0200 Subject: [PATCH 042/249] rm dockerfile + add citations --- nf_core/pipeline-template/.github/CONTRIBUTING.md | 1 - nf_core/pipeline-template/CITATIONS.md | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/CONTRIBUTING.md b/nf_core/pipeline-template/.github/CONTRIBUTING.md index 9afdd2987b..ecdda0f86b 100644 --- a/nf_core/pipeline-template/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/.github/CONTRIBUTING.md @@ -124,4 +124,3 @@ To get started: Devcontainer specs: - [DevContainer config](.devcontainer/devcontainer.json) -- [Dockerfile](.devcontainer/Dockerfile) diff --git a/nf_core/pipeline-template/CITATIONS.md b/nf_core/pipeline-template/CITATIONS.md index 740b045103..ceaba0cb5f 100644 --- a/nf_core/pipeline-template/CITATIONS.md +++ b/nf_core/pipeline-template/CITATIONS.md @@ -12,7 +12,10 @@ - [FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/) + > Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online]. Available online https://www.bioinformatics.babraham.ac.uk/projects/fastqc/. + - [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) + > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924. ## Software packaging/containerisation tools @@ -31,5 +34,8 @@ - [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241) + > Merkel, D. (2014). Docker: lightweight linux containers for consistent development and deployment. Linux Journal, 2014(239), 2. doi: 10.5555/2600239.2600241. + - [Singularity](https://pubmed.ncbi.nlm.nih.gov/28494014/) + > Kurtzer GM, Sochat V, Bauer MW. Singularity: Scientific containers for mobility of compute. PLoS One. 2017 May 11;12(5):e0177459. doi: 10.1371/journal.pone.0177459. eCollection 2017. PubMed PMID: 28494014; PubMed Central PMCID: PMC5426675. From 4d907907d9bddcf620d8f469888f324a9fa2ee2c Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 19 May 2023 16:06:53 +0200 Subject: [PATCH 043/249] improve startup time by localizing the imports --- nf_core/__main__.py | 81 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index e03fcbc67c..215f2d4d28 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -11,17 +11,7 @@ import rich_click as click import nf_core -import nf_core.bump_version -import nf_core.create -import nf_core.download -import nf_core.launch -import nf_core.licences -import nf_core.lint -import nf_core.list -import nf_core.modules -import nf_core.schema -import nf_core.subworkflows -import nf_core.sync +import nf_core.modules.modules_repo import nf_core.utils # Set up logging as the root logger @@ -163,6 +153,8 @@ def list(keywords, sort, json, show_archived): Checks the web for a list of nf-core pipelines with their latest releases. Shows which nf-core pipelines you have pulled locally and whether they are up to date. """ + import nf_core.list + stdout.print(nf_core.list.list_workflows(keywords, sort, json, show_archived)) @@ -207,6 +199,8 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all Run using a remote pipeline name (such as GitHub `user/repo` or a URL), a local pipeline directory or an ID from the nf-core web launch tool. """ + import nf_core.launch + launcher = nf_core.launch.Launch( pipeline, revision, command_only, params_in, params_out, save_all, show_hidden, url, id ) @@ -238,6 +232,8 @@ def download(pipeline, revision, outdir, compress, force, container, singularity Collects all files in a single archive and configures the downloaded workflow to use relative paths to the configs and singularity images. """ + import nf_core.download + dl = nf_core.download.DownloadWorkflow( pipeline, revision, outdir, compress, force, container, singularity_cache_only, parallel_downloads ) @@ -256,6 +252,7 @@ def licences(pipeline, json): Each of these is queried against the anaconda.org API to find the licence. Package name, version and licence is printed to the command line. """ + import nf_core.licences lic = nf_core.licences.WorkflowLicences(pipeline) lic.as_json = json @@ -289,6 +286,8 @@ def create(name, description, author, version, no_git, force, outdir, template_y Uses the nf-core template to make a skeleton Nextflow pipeline with all required files, boilerplate code and best-practices. """ + import nf_core.create + try: create_obj = nf_core.create.PipelineCreate( name, description, author, version, no_git, force, outdir, template_yaml, plain @@ -344,6 +343,8 @@ def lint(ctx, dir, release, fix, key, show_passed, fail_ignored, fail_warned, ma You can ignore tests using a file called [blue].nf-core.yml[/] [i](if you have a good reason!)[/]. See the documentation for details. """ + import nf_core.lint + import nf_core.utils # Check if pipeline directory is a pipeline try: @@ -460,6 +461,8 @@ def remote(ctx, keywords, json): """ List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ + import nf_core.modules + try: module_list = nf_core.modules.ModuleList( None, @@ -490,6 +493,8 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List modules installed locally in a pipeline """ + import nf_core.modules + try: module_list = nf_core.modules.ModuleList( dir, @@ -524,6 +529,8 @@ def install(ctx, tool, dir, prompt, force, sha): Fetches and installs module files from a remote repo e.g. nf-core/modules. """ + import nf_core.modules + try: module_install = nf_core.modules.ModuleInstall( dir, @@ -585,6 +592,8 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, update_d Fetches and updates module files from a remote repo e.g. nf-core/modules. """ + import nf_core.modules + try: module_install = nf_core.modules.ModuleUpdate( dir, @@ -626,6 +635,8 @@ def patch(ctx, tool, dir, remove): Checks if a module has been modified locally and creates a patch file describing how the module has changed from the remote version """ + import nf_core.modules + try: module_patch = nf_core.modules.ModulePatch( dir, @@ -657,6 +668,8 @@ def remove(ctx, dir, tool): """ Remove a module from a pipeline. """ + import nf_core.modules + try: module_remove = nf_core.modules.ModuleRemove( dir, @@ -710,6 +723,8 @@ def create_module( elif no_meta: has_meta = False + import nf_core.modules + # Run function try: module_create = nf_core.modules.ModuleCreate( @@ -739,6 +754,8 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): Given the name of a module, runs the Nextflow test command and automatically generate the required `test.yml` file based on the output files. """ + import nf_core.modules + try: meta_builder = nf_core.modules.ModulesTestYmlBuilder( module_name=tool, @@ -785,6 +802,8 @@ def lint( Test modules within a pipeline or a clone of the nf-core/modules repository. """ + import nf_core.modules + try: module_lint = nf_core.modules.ModuleLint( dir, @@ -837,6 +856,8 @@ def info(ctx, tool, dir): will print this usage info. If not, usage from the remote modules repo will be shown. """ + import nf_core.modules + try: module_info = nf_core.modules.ModuleInfo( dir, @@ -863,6 +884,8 @@ def bump_versions(ctx, tool, dir, all, show_all): Bump versions for one or more modules in a clone of the nf-core/modules repo. """ + import nf_core.modules + try: version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper( dir, @@ -891,6 +914,8 @@ def test_module(ctx, tool, no_prompts, pytest_args): Given the name of a module, runs the Nextflow test command. """ + import nf_core.modules + try: meta_builder = nf_core.modules.ModulesTest(tool, no_prompts, pytest_args) meta_builder.run() @@ -916,6 +941,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): If the specified directory is a clone of nf-core/modules, it creates or modifies files in 'subworkflows/', 'tests/subworkflows' and 'tests/config/pytest_modules.yml' """ + import nf_core.subworkflows # Run function try: @@ -944,6 +970,8 @@ def create_test_yml(ctx, subworkflow, run_tests, output, force, no_prompts): Given the name of a module, runs the Nextflow test command and automatically generate the required `test.yml` file based on the output files. """ + import nf_core.subworkflows + try: meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( subworkflow=subworkflow, @@ -979,6 +1007,8 @@ def remote(ctx, keywords, json): """ List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ + import nf_core.subworkflows + try: subworkflows_list = nf_core.subworkflows.SubworkflowList( None, @@ -1009,6 +1039,8 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List subworkflows installed locally in a pipeline """ + import nf_core.subworkflows + try: subworkflows_list = nf_core.subworkflows.SubworkflowList( dir, @@ -1046,6 +1078,8 @@ def info(ctx, tool, dir): will print this usage info. If not, usage from the remote subworkflows repo will be shown. """ + import nf_core.subworkflows + try: subworkflow_info = nf_core.subworkflows.SubworkflowInfo( dir, @@ -1072,6 +1106,8 @@ def test_subworkflow(ctx, subworkflow, no_prompts, pytest_args): Given the name of a subworkflow, runs the Nextflow test command. """ + import nf_core.subworkflows + try: meta_builder = nf_core.subworkflows.SubworkflowsTest(subworkflow, no_prompts, pytest_args) meta_builder.run() @@ -1102,6 +1138,8 @@ def install(ctx, subworkflow, dir, prompt, force, sha): Fetches and installs subworkflow files from a remote repo e.g. nf-core/modules. """ + import nf_core.subworkflows + try: subworkflow_install = nf_core.subworkflows.SubworkflowInstall( dir, @@ -1140,6 +1178,8 @@ def remote(ctx, keywords, json): """ List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ + import nf_core.subworkflows + try: subworkflow_list = nf_core.subworkflows.SubworkflowList( None, @@ -1171,6 +1211,8 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List subworkflows installed locally in a pipeline """ + import nf_core.subworkflows + try: subworkflow_list = nf_core.subworkflows.SubworkflowList( dir, @@ -1200,6 +1242,8 @@ def remove(ctx, dir, subworkflow): """ Remove a subworkflow from a pipeline. """ + import nf_core.subworkflows + try: module_remove = nf_core.subworkflows.SubworkflowRemove( dir, @@ -1256,6 +1300,8 @@ def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff, u Fetches and updates subworkflow files from a remote repo e.g. nf-core/modules. """ + import nf_core.subworkflows + try: subworkflow_install = nf_core.subworkflows.SubworkflowUpdate( dir, @@ -1304,6 +1350,8 @@ def validate(pipeline, params): This command takes such a file and validates it against the pipeline schema, checking whether all schema rules are satisfied. """ + import nf_core.schema + schema_obj = nf_core.schema.PipelineSchema() try: schema_obj.get_schema_path(pipeline) @@ -1348,6 +1396,8 @@ def build(dir, no_prompts, web_only, url): https://nf-co.re website where you can annotate and organise parameters. Listens for this to be completed and saves the updated schema. """ + import nf_core.schema + try: schema_obj = nf_core.schema.PipelineSchema() if schema_obj.build_schema(dir, no_prompts, web_only, url) is False: @@ -1374,6 +1424,8 @@ def lint(schema_path): If no schema path is provided, "nextflow_schema.json" will be used (if it exists). """ + import nf_core.schema + schema_obj = nf_core.schema.PipelineSchema() try: schema_obj.get_schema_path(schema_path) @@ -1415,6 +1467,7 @@ def docs(schema_path, output, format, force, columns): if not os.path.exists(schema_path): log.error("Could not find 'nextflow_schema.json' in current directory. Please specify a path.") sys.exit(1) + import nf_core.schema schema_obj = nf_core.schema.PipelineSchema() # Assume we're in a pipeline dir root if schema path not set @@ -1449,6 +1502,9 @@ def bump_version(new_version, dir, nextflow): As well as the pipeline version, you can also change the required version of Nextflow. """ + import nf_core.bump_version + import nf_core.utils + try: # Check if pipeline directory contains necessary files nf_core.utils.is_pipeline_directory(dir) @@ -1494,6 +1550,9 @@ def sync(dir, from_branch, pull_request, github_repository, username, template_y the pipeline. It is run automatically for all pipelines when ever a new release of [link=https://github.com/nf-core/tools]nf-core/tools[/link] (and the included template) is made. """ + import nf_core.sync + import nf_core.utils + # Check if pipeline directory contains necessary files nf_core.utils.is_pipeline_directory(dir) From 054321d40149357620254e710e83f50fc684c071 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 19 May 2023 18:01:12 +0200 Subject: [PATCH 044/249] replace pkg_resources --- nf_core/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/__init__.py b/nf_core/__init__.py index e333335280..d96be73f3d 100644 --- a/nf_core/__init__.py +++ b/nf_core/__init__.py @@ -3,6 +3,6 @@ Shouldn't do much, as everything is under subcommands. """ -import pkg_resources +import importlib.metadata as importlib_metadata -__version__ = pkg_resources.get_distribution("nf_core").version +__version__ = importlib_metadata.version(__name__) From 921c3055c9e7a923b05e05b261ee6bfd14cacfd3 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 19 May 2023 18:09:26 +0200 Subject: [PATCH 045/249] more granular imports --- nf_core/__main__.py | 193 ++++++++++++++++++++++---------------------- 1 file changed, 96 insertions(+), 97 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 215f2d4d28..d7685de493 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -10,16 +10,16 @@ import rich.traceback import rich_click as click -import nf_core -import nf_core.modules.modules_repo -import nf_core.utils +from nf_core import __version__ +from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE +from nf_core.utils import check_if_outdated, rich_force_colors, setup_nfcore_dir # Set up logging as the root logger # Submodules should all traverse back to this log = logging.getLogger() # Set up .nfcore directory for storing files between sessions -nf_core.utils.setup_nfcore_dir() +setup_nfcore_dir() # Set up nicer formatting of click cli help messages click.rich_click.MAX_WIDTH = 100 @@ -61,8 +61,8 @@ } # Set up rich stderr console -stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()) -stdout = rich.console.Console(force_terminal=nf_core.utils.rich_force_colors()) +stderr = rich.console.Console(stderr=True, force_terminal=rich_force_colors()) +stdout = rich.console.Console(force_terminal=rich_force_colors()) # Set up the rich traceback rich.traceback.install(console=stderr, width=200, word_wrap=True, extra_lines=1) @@ -78,11 +78,11 @@ def run_nf_core(): stderr.print(r"[blue] | \| | \__, \__/ | \ |___ [green]\`-._,-`-,", highlight=False) stderr.print("[green] `._,._,'\n", highlight=False) stderr.print( - f"[grey39] nf-core/tools version {nf_core.__version__} - [link=https://nf-co.re]https://nf-co.re[/]", + f"[grey39] nf-core/tools version {__version__} - [link=https://nf-co.re]https://nf-co.re[/]", highlight=False, ) try: - is_outdated, _, remote_vers = nf_core.utils.check_if_outdated() + is_outdated, _, remote_vers = check_if_outdated() if is_outdated: stderr.print( f"[bold bright_yellow] There is a new version of nf-core/tools available! ({remote_vers})", @@ -96,7 +96,7 @@ def run_nf_core(): @click.group(context_settings=dict(help_option_names=["-h", "--help"])) -@click.version_option(nf_core.__version__) +@click.version_option(__version__) @click.option("-v", "--verbose", is_flag=True, default=False, help="Print verbose output to the console.") @click.option("--hide-progress", is_flag=True, default=False, help="Don't show progress bars.") @click.option("-l", "--log-file", help="Save a verbose log to a file.", metavar="") @@ -114,7 +114,7 @@ def nf_core_cli(ctx, verbose, hide_progress, log_file): log.addHandler( rich.logging.RichHandler( level=logging.DEBUG if verbose else logging.INFO, - console=rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()), + console=rich.console.Console(stderr=True, force_terminal=rich_force_colors()), show_time=False, show_path=verbose, # True if verbose, false otherwise markup=True, @@ -153,9 +153,9 @@ def list(keywords, sort, json, show_archived): Checks the web for a list of nf-core pipelines with their latest releases. Shows which nf-core pipelines you have pulled locally and whether they are up to date. """ - import nf_core.list + from nf_core.list import list_workflows - stdout.print(nf_core.list.list_workflows(keywords, sort, json, show_archived)) + stdout.print(list_workflows(keywords, sort, json, show_archived)) # nf-core launch @@ -199,11 +199,9 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all Run using a remote pipeline name (such as GitHub `user/repo` or a URL), a local pipeline directory or an ID from the nf-core web launch tool. """ - import nf_core.launch + from nf_core.launch import Launch - launcher = nf_core.launch.Launch( - pipeline, revision, command_only, params_in, params_out, save_all, show_hidden, url, id - ) + launcher = Launch(pipeline, revision, command_only, params_in, params_out, save_all, show_hidden, url, id) if not launcher.launch_pipeline(): sys.exit(1) @@ -232,9 +230,9 @@ def download(pipeline, revision, outdir, compress, force, container, singularity Collects all files in a single archive and configures the downloaded workflow to use relative paths to the configs and singularity images. """ - import nf_core.download + from nf_core.download import DownloadWorkflow - dl = nf_core.download.DownloadWorkflow( + dl = DownloadWorkflow( pipeline, revision, outdir, compress, force, container, singularity_cache_only, parallel_downloads ) dl.download_workflow() @@ -252,9 +250,9 @@ def licences(pipeline, json): Each of these is queried against the anaconda.org API to find the licence. Package name, version and licence is printed to the command line. """ - import nf_core.licences + from nf_core.licences import WorkflowLicences - lic = nf_core.licences.WorkflowLicences(pipeline) + lic = WorkflowLicences(pipeline) lic.as_json = json try: stdout.print(lic.run_licences()) @@ -286,12 +284,10 @@ def create(name, description, author, version, no_git, force, outdir, template_y Uses the nf-core template to make a skeleton Nextflow pipeline with all required files, boilerplate code and best-practices. """ - import nf_core.create + from nf_core.create import PipelineCreate try: - create_obj = nf_core.create.PipelineCreate( - name, description, author, version, no_git, force, outdir, template_yaml, plain - ) + create_obj = PipelineCreate(name, description, author, version, no_git, force, outdir, template_yaml, plain) create_obj.init_pipeline() except UserWarning as e: log.error(e) @@ -343,19 +339,19 @@ def lint(ctx, dir, release, fix, key, show_passed, fail_ignored, fail_warned, ma You can ignore tests using a file called [blue].nf-core.yml[/] [i](if you have a good reason!)[/]. See the documentation for details. """ - import nf_core.lint - import nf_core.utils + from nf_core.lint import run_linting + from nf_core.utils import is_pipeline_directory # Check if pipeline directory is a pipeline try: - nf_core.utils.is_pipeline_directory(dir) + is_pipeline_directory(dir) except UserWarning as e: log.error(e) sys.exit(1) # Run the lint tests! try: - lint_obj, module_lint_obj = nf_core.lint.run_linting( + lint_obj, module_lint_obj = run_linting( dir, release, fix, @@ -384,7 +380,7 @@ def lint(ctx, dir, release, fix, key, show_passed, fail_ignored, fail_warned, ma "-g", "--git-remote", type=str, - default=nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE, + default=NF_CORE_MODULES_REMOTE, help="Remote git repo to fetch files from", ) @click.option("-b", "--branch", type=str, default=None, help="Branch of git repository hosting modules.") @@ -416,7 +412,7 @@ def modules(ctx, git_remote, branch, no_pull): "-g", "--git-remote", type=str, - default=nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE, + default=NF_CORE_MODULES_REMOTE, help="Remote git repo to fetch files from", ) @click.option("-b", "--branch", type=str, default=None, help="Branch of git repository hosting modules.") @@ -461,10 +457,10 @@ def remote(ctx, keywords, json): """ List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ - import nf_core.modules + from nf_core.modules import ModuleList try: - module_list = nf_core.modules.ModuleList( + module_list = ModuleList( None, True, ctx.obj["modules_repo_url"], @@ -493,10 +489,10 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List modules installed locally in a pipeline """ - import nf_core.modules + from nf_core.modules import ModuleList try: - module_list = nf_core.modules.ModuleList( + module_list = ModuleList( dir, False, ctx.obj["modules_repo_url"], @@ -529,10 +525,10 @@ def install(ctx, tool, dir, prompt, force, sha): Fetches and installs module files from a remote repo e.g. nf-core/modules. """ - import nf_core.modules + from nf_core.modules import ModuleInstall try: - module_install = nf_core.modules.ModuleInstall( + module_install = ModuleInstall( dir, force, prompt, @@ -592,10 +588,10 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, update_d Fetches and updates module files from a remote repo e.g. nf-core/modules. """ - import nf_core.modules + from nf_core.modules import ModuleUpdate try: - module_install = nf_core.modules.ModuleUpdate( + module_install = ModuleUpdate( dir, force, prompt, @@ -635,10 +631,10 @@ def patch(ctx, tool, dir, remove): Checks if a module has been modified locally and creates a patch file describing how the module has changed from the remote version """ - import nf_core.modules + from nf_core.modules import ModulePatch try: - module_patch = nf_core.modules.ModulePatch( + module_patch = ModulePatch( dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], @@ -668,10 +664,10 @@ def remove(ctx, dir, tool): """ Remove a module from a pipeline. """ - import nf_core.modules + from nf_core.modules import ModuleRemove try: - module_remove = nf_core.modules.ModuleRemove( + module_remove = ModuleRemove( dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], @@ -723,11 +719,11 @@ def create_module( elif no_meta: has_meta = False - import nf_core.modules + from nf_core.modules import ModuleCreate # Run function try: - module_create = nf_core.modules.ModuleCreate( + module_create = ModuleCreate( dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template ) module_create.create() @@ -754,10 +750,10 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): Given the name of a module, runs the Nextflow test command and automatically generate the required `test.yml` file based on the output files. """ - import nf_core.modules + from nf_core.modules import ModulesTestYmlBuilder try: - meta_builder = nf_core.modules.ModulesTestYmlBuilder( + meta_builder = ModulesTestYmlBuilder( module_name=tool, run_tests=run_tests, test_yml_output_path=output, @@ -802,10 +798,11 @@ def lint( Test modules within a pipeline or a clone of the nf-core/modules repository. """ - import nf_core.modules + from nf_core.modules import ModuleLint + from nf_core.modules.lint import ModuleLintException try: - module_lint = nf_core.modules.ModuleLint( + module_lint = ModuleLint( dir, fail_warned=fail_warned, remote_url=ctx.obj["modules_repo_url"], @@ -825,7 +822,7 @@ def lint( ) if len(module_lint.failed) > 0: sys.exit(1) - except nf_core.modules.lint.ModuleLintException as e: + except ModuleLintException as e: log.error(e) sys.exit(1) except (UserWarning, LookupError) as e: @@ -856,10 +853,10 @@ def info(ctx, tool, dir): will print this usage info. If not, usage from the remote modules repo will be shown. """ - import nf_core.modules + from nf_core.modules import ModuleInfo try: - module_info = nf_core.modules.ModuleInfo( + module_info = ModuleInfo( dir, tool, ctx.obj["modules_repo_url"], @@ -884,17 +881,18 @@ def bump_versions(ctx, tool, dir, all, show_all): Bump versions for one or more modules in a clone of the nf-core/modules repo. """ - import nf_core.modules + from nf_core.modules.bump_versions import ModuleVersionBumper + from nf_core.modules.modules_utils import ModuleException try: - version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper( + version_bumper = ModuleVersionBumper( dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) version_bumper.bump_versions(module=tool, all_modules=all, show_uptodate=show_all) - except nf_core.modules.modules_utils.ModuleException as e: + except ModuleException as e: log.error(e) sys.exit(1) except (UserWarning, LookupError) as e: @@ -914,10 +912,10 @@ def test_module(ctx, tool, no_prompts, pytest_args): Given the name of a module, runs the Nextflow test command. """ - import nf_core.modules + from nf_core.modules import ModulesTest try: - meta_builder = nf_core.modules.ModulesTest(tool, no_prompts, pytest_args) + meta_builder = ModulesTest(tool, no_prompts, pytest_args) meta_builder.run() except (UserWarning, LookupError) as e: log.critical(e) @@ -941,11 +939,11 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): If the specified directory is a clone of nf-core/modules, it creates or modifies files in 'subworkflows/', 'tests/subworkflows' and 'tests/config/pytest_modules.yml' """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowCreate # Run function try: - subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force) + subworkflow_create = SubworkflowCreate(dir, subworkflow, author, force) subworkflow_create.create() except UserWarning as e: log.critical(e) @@ -970,10 +968,10 @@ def create_test_yml(ctx, subworkflow, run_tests, output, force, no_prompts): Given the name of a module, runs the Nextflow test command and automatically generate the required `test.yml` file based on the output files. """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowTestYmlBuilder try: - meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( + meta_builder = SubworkflowTestYmlBuilder( subworkflow=subworkflow, run_tests=run_tests, test_yml_output_path=output, @@ -1007,10 +1005,10 @@ def remote(ctx, keywords, json): """ List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowList try: - subworkflows_list = nf_core.subworkflows.SubworkflowList( + subworkflows_list = SubworkflowList( None, True, ctx.obj["modules_repo_url"], @@ -1039,10 +1037,10 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List subworkflows installed locally in a pipeline """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowList try: - subworkflows_list = nf_core.subworkflows.SubworkflowList( + subworkflows_list = SubworkflowList( dir, False, ctx.obj["modules_repo_url"], @@ -1078,10 +1076,10 @@ def info(ctx, tool, dir): will print this usage info. If not, usage from the remote subworkflows repo will be shown. """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowInfo try: - subworkflow_info = nf_core.subworkflows.SubworkflowInfo( + subworkflow_info = SubworkflowInfo( dir, tool, ctx.obj["modules_repo_url"], @@ -1106,10 +1104,10 @@ def test_subworkflow(ctx, subworkflow, no_prompts, pytest_args): Given the name of a subworkflow, runs the Nextflow test command. """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowsTest try: - meta_builder = nf_core.subworkflows.SubworkflowsTest(subworkflow, no_prompts, pytest_args) + meta_builder = SubworkflowsTest(subworkflow, no_prompts, pytest_args) meta_builder.run() except (UserWarning, LookupError) as e: log.critical(e) @@ -1138,10 +1136,10 @@ def install(ctx, subworkflow, dir, prompt, force, sha): Fetches and installs subworkflow files from a remote repo e.g. nf-core/modules. """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowInstall try: - subworkflow_install = nf_core.subworkflows.SubworkflowInstall( + subworkflow_install = SubworkflowInstall( dir, force, prompt, @@ -1178,10 +1176,10 @@ def remote(ctx, keywords, json): """ List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowList try: - subworkflow_list = nf_core.subworkflows.SubworkflowList( + subworkflow_list = SubworkflowList( None, True, ctx.obj["modules_repo_url"], @@ -1211,10 +1209,10 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List subworkflows installed locally in a pipeline """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowList try: - subworkflow_list = nf_core.subworkflows.SubworkflowList( + subworkflow_list = SubworkflowList( dir, False, ctx.obj["modules_repo_url"], @@ -1242,10 +1240,10 @@ def remove(ctx, dir, subworkflow): """ Remove a subworkflow from a pipeline. """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowRemove try: - module_remove = nf_core.subworkflows.SubworkflowRemove( + module_remove = SubworkflowRemove( dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], @@ -1300,10 +1298,10 @@ def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff, u Fetches and updates subworkflow files from a remote repo e.g. nf-core/modules. """ - import nf_core.subworkflows + from nf_core.subworkflows import SubworkflowUpdate try: - subworkflow_install = nf_core.subworkflows.SubworkflowUpdate( + subworkflow_install = SubworkflowUpdate( dir, force, prompt, @@ -1350,9 +1348,9 @@ def validate(pipeline, params): This command takes such a file and validates it against the pipeline schema, checking whether all schema rules are satisfied. """ - import nf_core.schema + from nf_core.schema import PipelineSchema - schema_obj = nf_core.schema.PipelineSchema() + schema_obj = PipelineSchema() try: schema_obj.get_schema_path(pipeline) # Load and check schema @@ -1396,10 +1394,10 @@ def build(dir, no_prompts, web_only, url): https://nf-co.re website where you can annotate and organise parameters. Listens for this to be completed and saves the updated schema. """ - import nf_core.schema + from nf_core.schema import PipelineSchema try: - schema_obj = nf_core.schema.PipelineSchema() + schema_obj = PipelineSchema() if schema_obj.build_schema(dir, no_prompts, web_only, url) is False: sys.exit(1) except (UserWarning, AssertionError) as e: @@ -1424,9 +1422,9 @@ def lint(schema_path): If no schema path is provided, "nextflow_schema.json" will be used (if it exists). """ - import nf_core.schema + from nf_core.schema import PipelineSchema - schema_obj = nf_core.schema.PipelineSchema() + schema_obj = PipelineSchema() try: schema_obj.get_schema_path(schema_path) schema_obj.load_lint_schema() @@ -1467,9 +1465,10 @@ def docs(schema_path, output, format, force, columns): if not os.path.exists(schema_path): log.error("Could not find 'nextflow_schema.json' in current directory. Please specify a path.") sys.exit(1) - import nf_core.schema - schema_obj = nf_core.schema.PipelineSchema() + from nf_core.schema import PipelineSchema + + schema_obj = PipelineSchema() # Assume we're in a pipeline dir root if schema path not set schema_obj.get_schema_path(schema_path) schema_obj.load_schema() @@ -1502,22 +1501,22 @@ def bump_version(new_version, dir, nextflow): As well as the pipeline version, you can also change the required version of Nextflow. """ - import nf_core.bump_version - import nf_core.utils + from nf_core.bump_version import bump_nextflow_version, bump_pipeline_version + from nf_core.utils import Pipeline, is_pipeline_directory try: # Check if pipeline directory contains necessary files - nf_core.utils.is_pipeline_directory(dir) + is_pipeline_directory(dir) # Make a pipeline object and load config etc - pipeline_obj = nf_core.utils.Pipeline(dir) + pipeline_obj = Pipeline(dir) pipeline_obj._load() # Bump the pipeline version number if not nextflow: - nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version) + bump_pipeline_version(pipeline_obj, new_version) else: - nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version) + bump_nextflow_version(pipeline_obj, new_version) except UserWarning as e: log.error(e) sys.exit(1) @@ -1550,17 +1549,17 @@ def sync(dir, from_branch, pull_request, github_repository, username, template_y the pipeline. It is run automatically for all pipelines when ever a new release of [link=https://github.com/nf-core/tools]nf-core/tools[/link] (and the included template) is made. """ - import nf_core.sync - import nf_core.utils + from nf_core.sync import PipelineSync, PullRequestException, SyncException + from nf_core.utils import is_pipeline_directory # Check if pipeline directory contains necessary files - nf_core.utils.is_pipeline_directory(dir) + is_pipeline_directory(dir) # Sync the given pipeline dir - sync_obj = nf_core.sync.PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml) + sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml) try: sync_obj.sync() - except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: + except (SyncException, PullRequestException) as e: log.error(e) sys.exit(1) From 48f19965fbf8a1e6a5d0f038eaa47fcd7bc131f3 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 19 May 2023 19:10:55 +0200 Subject: [PATCH 046/249] remove duplicated commands --- nf_core/__main__.py | 77 +++------------------------------------------ 1 file changed, 5 insertions(+), 72 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d7685de493..735eb99e04 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1008,14 +1008,15 @@ def remote(ctx, keywords, json): from nf_core.subworkflows import SubworkflowList try: - subworkflows_list = SubworkflowList( + subworkflow_list = SubworkflowList( None, True, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) - stdout.print(subworkflows_list.list_subworkflows(keywords, json)) + + stdout.print(subworkflow_list.list_components(keywords, json)) except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) @@ -1040,14 +1041,14 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin from nf_core.subworkflows import SubworkflowList try: - subworkflows_list = SubworkflowList( + subworkflow_list = SubworkflowList( dir, False, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) - stdout.print(subworkflows_list.list_subworkflows(keywords, json)) + stdout.print(subworkflow_list.list_components(keywords, json)) except (UserWarning, LookupError) as e: log.error(e) sys.exit(1) @@ -1157,74 +1158,6 @@ def install(ctx, subworkflow, dir, prompt, force, sha): sys.exit(1) -# nf-core subworkflows list subcommands -@subworkflows.group() -@click.pass_context -def list(ctx): - """ - List modules in a local pipeline or remote repository. - """ - pass - - -# nf-core subworkflows list remote -@list.command() -@click.pass_context -@click.argument("keywords", required=False, nargs=-1, metavar="") -@click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -def remote(ctx, keywords, json): - """ - List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. - """ - from nf_core.subworkflows import SubworkflowList - - try: - subworkflow_list = SubworkflowList( - None, - True, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], - ctx.obj["modules_repo_no_pull"], - ) - - stdout.print(subworkflow_list.list_components(keywords, json)) - except (UserWarning, LookupError) as e: - log.critical(e) - sys.exit(1) - - -# nf-core subworkflows list local -@list.command() -@click.pass_context -@click.argument("keywords", required=False, nargs=-1, metavar="") -@click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -@click.option( - "-d", - "--dir", - type=click.Path(exists=True), - default=".", - help=r"Pipeline directory. [dim]\[default: Current working directory][/]", -) -def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin - """ - List subworkflows installed locally in a pipeline - """ - from nf_core.subworkflows import SubworkflowList - - try: - subworkflow_list = SubworkflowList( - dir, - False, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], - ctx.obj["modules_repo_no_pull"], - ) - stdout.print(subworkflow_list.list_components(keywords, json)) - except (UserWarning, LookupError) as e: - log.error(e) - sys.exit(1) - - # nf-core subworkflows remove @subworkflows.command() @click.pass_context From 011199ec28de02c0d7cb859ee99ada22ed7e8b57 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 22 May 2023 13:31:14 +0200 Subject: [PATCH 047/249] add missing return values --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 36c39db50a..3fa72d8b53 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -64,7 +64,7 @@ def check_if_outdated(current_version=None, remote_version=None, source_url="htt """ # Exit immediately if disabled via ENV var if os.environ.get("NFCORE_NO_VERSION_CHECK", False): - return True + return (True, "", "") # Set and clean up the current version string if current_version is None: current_version = nf_core.__version__ From c5f67b00e872b0e92b531198b9411ca54e552e57 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 22 May 2023 13:31:25 +0200 Subject: [PATCH 048/249] fix mocked function --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 58c4525a76..0a6b37144d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -21,7 +21,7 @@ def test_header(mock_cli): @mock.patch("nf_core.__main__.nf_core_cli") -@mock.patch("nf_core.utils.check_if_outdated", return_value=(True, None, "dummy_version")) +@mock.patch("nf_core.__main__.check_if_outdated", return_value=(True, None, "dummy_version")) def test_header_outdated(mock_check_outdated, mock_nf_core_cli, capsys): """Check cli notifies the user when nf_core is outdated""" nf_core.__main__.run_nf_core() From f9d07efc3014df6085d1455dffa19745b50c5667 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 5 Oct 2022 17:04:29 +0200 Subject: [PATCH 049/249] Add -t / --tower option to 'nf-core download'. --- nf_core/__main__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 735eb99e04..725bcfc895 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -215,6 +215,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all "-x", "--compress", type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]), help="Archive compression type" ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") +@click.option("-t", "--tower", is_flag=True, default=False, help="Customize download for sequeralabs® Nextflow Tower") @click.option( "-c", "--container", type=click.Choice(["none", "singularity"]), help="Download software container images" ) @@ -223,7 +224,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all help="Don't / do copy images to the output directory and set 'singularity.cacheDir' in workflow", ) @click.option("-p", "--parallel-downloads", type=int, default=4, help="Number of parallel image downloads") -def download(pipeline, revision, outdir, compress, force, container, singularity_cache_only, parallel_downloads): +def download(pipeline, revision, outdir, compress, force, tower, container, singularity_cache_only, parallel_downloads): """ Download a pipeline, nf-core/configs and pipeline singularity images. @@ -233,7 +234,7 @@ def download(pipeline, revision, outdir, compress, force, container, singularity from nf_core.download import DownloadWorkflow dl = DownloadWorkflow( - pipeline, revision, outdir, compress, force, container, singularity_cache_only, parallel_downloads + pipeline, revision, outdir, compress, force, tower, container, singularity_cache_only, parallel_downloads ) dl.download_workflow() From e94dce06c071f0b9fc957ec76ad7fc64bb29630f Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Mon, 5 Dec 2022 18:04:40 +0100 Subject: [PATCH 050/249] Intermediate commit --- nf_core/download.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/download.py b/nf_core/download.py index cd36c65c4a..8ecd2fd85c 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -73,6 +73,7 @@ class DownloadWorkflow: pipeline (str): A nf-core pipeline name. revision (str): The workflow revision to download, like `1.0`. Defaults to None. singularity (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False. + tower (bool): Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. outdir (str): Path to the local download directory. Defaults to None. """ @@ -83,6 +84,7 @@ def __init__( outdir=None, compress_type=None, force=False, + tower=False, container=None, singularity_cache_only=False, parallel_downloads=4, @@ -93,6 +95,7 @@ def __init__( self.output_filename = None self.compress_type = compress_type self.force = force + self.tower = tower self.container = container self.singularity_cache_only = singularity_cache_only self.parallel_downloads = parallel_downloads From cba336b4970635703736fd8bc20995c753d6253b Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 15 Feb 2023 16:04:41 +0100 Subject: [PATCH 051/249] Implement logic for the Tower download in DownloadWorkflow:download_workflow() --- nf_core/download.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index 8ecd2fd85c..c3451818e8 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -135,7 +135,10 @@ def download_workflow(self): summary_log.append(f"Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {os.environ['NXF_SINGULARITY_CACHEDIR']}") # Set an output filename now that we have the outdir - if self.compress_type is not None: + if self.tower: + self.output_filename = f"{self.outdir}.git" + summary_log.append(f"Output file (Tower enabled): '{self.output_filename}'") + elif self.compress_type is not None: self.output_filename = f"{self.outdir}.{self.compress_type}" summary_log.append(f"Output file: '{self.output_filename}'") else: @@ -160,6 +163,13 @@ def download_workflow(self): # Summary log log.info("Saving '{}'\n {}".format(self.pipeline, "\n ".join(summary_log))) + # Actually download the workflow + if not self.tower: + self.download_workflow_classic() + else: + self.download_workflow_tower() + + def download_workflow_classic(self): # Download the pipeline files log.info("Downloading workflow files from GitHub") self.download_wf_files() @@ -188,6 +198,15 @@ def download_workflow(self): log.info("Compressing download..") self.compress_download() + def download_workflow_tower(self): + # Create a bare-cloned git repository of the workflow that includes the configs + log.info("Cloning workflow files from GitHub") + self.clone_wf_files() + + # Download the centralised configs + log.info("Downloading centralised configs from GitHub") + self.download_configs() + def prompt_pipeline_name(self): """Prompt for the pipeline name if not set with a flag""" From 15c0588a88ac50b98c89d12dbe8ed40bff44a369 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 15 Feb 2023 18:40:22 +0100 Subject: [PATCH 052/249] Extend ModulesRepo:setup_local_repo() with a cache_only bool, so we can use ModuleRepo as superclass to the new WorkflowRepo. --- nf_core/modules/modules_repo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 5f77148867..0f5db4bc52 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -11,7 +11,7 @@ import nf_core.modules.modules_json import nf_core.modules.modules_utils -from nf_core.utils import NFCORE_DIR, load_tools_config +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config log = logging.getLogger(__name__) @@ -166,7 +166,7 @@ def verify_sha(self, prompt, sha): return True - def setup_local_repo(self, remote, branch, hide_progress=True): + def setup_local_repo(self, remote, branch, hide_progress=True, cache_only=False): """ Sets up the local git repository. If the repository has been cloned previously, it returns a git.Repo object of that clone. Otherwise it tries to clone the repository from @@ -177,7 +177,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True): branch (str): name of branch to use Sets self.repo """ - self.local_repo_dir = os.path.join(NFCORE_DIR, self.fullname) + self.local_repo_dir = os.path.join(NFCORE_DIR if not cache_only else NFCORE_CACHE_DIR, self.fullname) try: if not os.path.exists(self.local_repo_dir): try: From 2be88c4d475f4388a3ab360dcb5ec8ddfb43ffb0 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 15 Feb 2023 20:11:22 +0100 Subject: [PATCH 053/249] Create WorkflowRepo subclass of ModuleRepo and initialise local clone. --- nf_core/__main__.py | 2 +- nf_core/download.py | 41 +++++++++++++++++++++++++++++---- nf_core/modules/modules_repo.py | 4 ++-- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 725bcfc895..521454eb99 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -215,7 +215,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all "-x", "--compress", type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]), help="Archive compression type" ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") -@click.option("-t", "--tower", is_flag=True, default=False, help="Customize download for sequeralabs® Nextflow Tower") +@click.option("-t", "--tower", is_flag=True, default=False, help="Customize download for seqeralabs® Nextflow Tower") @click.option( "-c", "--container", type=click.Choice(["none", "singularity"]), help="Download software container images" ) diff --git a/nf_core/download.py b/nf_core/download.py index c3451818e8..5c2ff1607d 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -3,6 +3,7 @@ from __future__ import print_function import concurrent.futures +from git import Repo import io import logging import os @@ -23,6 +24,7 @@ import nf_core import nf_core.list import nf_core.utils +from nf_core.modules import ModulesRepo # to create subclass WorkflowRepo log = logging.getLogger(__name__) stderr = rich.console.Console( @@ -137,13 +139,15 @@ def download_workflow(self): # Set an output filename now that we have the outdir if self.tower: self.output_filename = f"{self.outdir}.git" - summary_log.append(f"Output file (Tower enabled): '{self.output_filename}'") + summary_log.append(f"Output file: '{self.output_filename}'") elif self.compress_type is not None: self.output_filename = f"{self.outdir}.{self.compress_type}" summary_log.append(f"Output file: '{self.output_filename}'") else: summary_log.append(f"Output directory: '{self.outdir}'") + summary_log.append(f"Enabled for seqeralabs® Nextflow Tower: '{self.tower}'") + # Check that the outdir doesn't already exist if os.path.exists(self.outdir): if not self.force: @@ -170,6 +174,7 @@ def download_workflow(self): self.download_workflow_tower() def download_workflow_classic(self): + """Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner.""" # Download the pipeline files log.info("Downloading workflow files from GitHub") self.download_wf_files() @@ -199,9 +204,10 @@ def download_workflow_classic(self): self.compress_download() def download_workflow_tower(self): - # Create a bare-cloned git repository of the workflow that includes the configs + """Create a bare-cloned git repository of the workflow that includes the configurations, such it can be launched with `tw launch` as file:/ pipeline""" log.info("Cloning workflow files from GitHub") - self.clone_wf_files() + + self.workflow_repo = WorkflowRepo(remote_url=f"git@github.com:{self.pipeline}.git", branch=self.revision) # Download the centralised configs log.info("Downloading centralised configs from GitHub") @@ -816,5 +822,32 @@ def compress_download(self): log.debug(f"Deleting uncompressed files: '{self.outdir}'") shutil.rmtree(self.outdir) - # Caclualte md5sum for output file + # Calculate md5sum for output file log.info(f"MD5 checksum for '{self.output_filename}': [blue]{nf_core.utils.file_md5(self.output_filename)}[/]") + + +class WorkflowRepo(ModulesRepo): + """ + An object to store details about a locally cached workflow repository. + + Important Attributes: + fullname: The full name of the repository, ``nf-core/{self.pipelinename}``. + local_repo_dir (str): The local directory, where the workflow is cloned into. Defaults to ``$HOME/.cache/nf-core/nf-core/{self.pipeline}``. + + """ + + def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=False, in_cache=True): + """ + Initializes the object and clones the workflows git repository if it is not already present + + Args: + remote_url (str, optional): The URL of the remote repository. Defaults to None. + branch (str, optional): The branch to clone. Defaults to None. + no_pull (bool, optional): Whether to skip the pull step. Defaults to False. + hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. + in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. + """ + self.remote_url = remote_url + self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) + + self.setup_local_repo(remote_url, branch, hide_progress, in_cache=in_cache) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 0f5db4bc52..23f62bdee2 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -166,7 +166,7 @@ def verify_sha(self, prompt, sha): return True - def setup_local_repo(self, remote, branch, hide_progress=True, cache_only=False): + def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): """ Sets up the local git repository. If the repository has been cloned previously, it returns a git.Repo object of that clone. Otherwise it tries to clone the repository from @@ -177,7 +177,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True, cache_only=False) branch (str): name of branch to use Sets self.repo """ - self.local_repo_dir = os.path.join(NFCORE_DIR if not cache_only else NFCORE_CACHE_DIR, self.fullname) + self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname) try: if not os.path.exists(self.local_repo_dir): try: From d72169df6ed25c2f80a8c1f2ed2872242a060332 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 15 Feb 2023 23:03:02 +0100 Subject: [PATCH 054/249] TypeError: HEAD is a detached symbolic reference as it points to ... --- nf_core/download.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 5c2ff1607d..74b8abf3d8 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -205,13 +205,13 @@ def download_workflow_classic(self): def download_workflow_tower(self): """Create a bare-cloned git repository of the workflow that includes the configurations, such it can be launched with `tw launch` as file:/ pipeline""" - log.info("Cloning workflow files from GitHub") + log.info("Collecting workflow from GitHub") self.workflow_repo = WorkflowRepo(remote_url=f"git@github.com:{self.pipeline}.git", branch=self.revision) + import pbb - # Download the centralised configs + pdb.set_trace() log.info("Downloading centralised configs from GitHub") - self.download_configs() def prompt_pipeline_name(self): """Prompt for the pipeline name if not set with a flag""" @@ -851,3 +851,14 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) self.setup_local_repo(remote_url, branch, hide_progress, in_cache=in_cache) + + @property + def active_branch(self): + """ + In ModuleRepo.setup_local_repo(), self.repo.active_branch.tracking_branch() is called in line 227. + For a WorkflowRepo, this raises a TypeError ``HEAD is a detached symbolic reference as it points to {commit hash}`` + + This property shadows the call and seemed the cleanest solution to prevent excessive code duplication. + Otherwise, I would have needed to define a setup_local_repo() method for the WorkflowRepo class. + """ + pass # TODO From a6b34926c3c4661a50a3000f273c3c0fb78ceec1 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 21 Feb 2023 15:11:00 +0100 Subject: [PATCH 055/249] Split history ./modules/modules_repo.py to synced_repo.py --- nf_core/{modules/modules_repo.py => temp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nf_core/{modules/modules_repo.py => temp} (100%) diff --git a/nf_core/modules/modules_repo.py b/nf_core/temp similarity index 100% rename from nf_core/modules/modules_repo.py rename to nf_core/temp From 8984a6391fd0794e107d7696d3eedfd84e8f5918 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 21 Feb 2023 15:11:00 +0100 Subject: [PATCH 056/249] Split history ./modules/modules_repo.py to synced_repo.py --- nf_core/synced_repo.py | 498 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 498 insertions(+) create mode 100644 nf_core/synced_repo.py diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py new file mode 100644 index 0000000000..23f62bdee2 --- /dev/null +++ b/nf_core/synced_repo.py @@ -0,0 +1,498 @@ +import filecmp +import logging +import os +import shutil +from pathlib import Path + +import git +import rich +import rich.progress +from git.exc import GitCommandError, InvalidGitRepositoryError + +import nf_core.modules.modules_json +import nf_core.modules.modules_utils +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config + +log = logging.getLogger(__name__) + +# Constants for the nf-core/modules repo used throughout the module files +NF_CORE_MODULES_NAME = "nf-core" +NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" +NF_CORE_MODULES_DEFAULT_BRANCH = "master" + + +class RemoteProgressbar(git.RemoteProgress): + """ + An object to create a progressbar for when doing an operation with the remote. + Note that an initialized rich Progress (progress bar) object must be past + during initialization. + """ + + def __init__(self, progress_bar, repo_name, remote_url, operation): + """ + Initializes the object and adds a task to the progressbar passed as 'progress_bar' + + Args: + progress_bar (rich.progress.Progress): A rich progress bar object + repo_name (str): Name of the repository the operation is performed on + remote_url (str): Git URL of the repository the operation is performed on + operation (str): The operation performed on the repository, i.e. 'Pulling', 'Cloning' etc. + """ + super().__init__() + self.progress_bar = progress_bar + self.tid = self.progress_bar.add_task( + f"{operation} from [bold green]'{repo_name}'[/bold green] ([link={remote_url}]{remote_url}[/link])", + start=False, + state="Waiting for response", + ) + + def update(self, op_code, cur_count, max_count=None, message=""): + """ + Overrides git.RemoteProgress.update. + Called every time there is a change in the remote operation + """ + if not self.progress_bar.tasks[self.tid].started: + self.progress_bar.start_task(self.tid) + self.progress_bar.update( + self.tid, total=max_count, completed=cur_count, state=f"{cur_count / max_count * 100:.1f}%" + ) + + +class ModulesRepo: + """ + An object to store details about the repository being used for modules. + + Used by the `nf-core modules` top-level command with -r and -b flags, + so that this can be used in the same way by all sub-commands. + + We keep track of the pull-status of the different installed repos in + the static variable local_repo_status. This is so we don't need to + pull a remote several times in one command. + """ + + local_repo_statuses = {} + no_pull_global = False + + @staticmethod + def local_repo_synced(repo_name): + """ + Checks whether a local repo has been cloned/pull in the current session + """ + return ModulesRepo.local_repo_statuses.get(repo_name, False) + + @staticmethod + def update_local_repo_status(repo_name, up_to_date): + """ + Updates the clone/pull status of a local repo + """ + ModulesRepo.local_repo_statuses[repo_name] = up_to_date + + @staticmethod + def get_remote_branches(remote_url): + """ + Get all branches from a remote repository + + Args: + remote_url (str): The git url to the remote repository + + Returns: + (set[str]): All branches found in the remote + """ + try: + unparsed_branches = git.Git().ls_remote(remote_url) + except git.GitCommandError: + raise LookupError(f"Was unable to fetch branches from '{remote_url}'") + else: + branches = {} + for branch_info in unparsed_branches.split("\n"): + sha, name = branch_info.split("\t") + if name != "HEAD": + # The remote branches are shown as 'ref/head/branch' + branch_name = Path(name).stem + branches[sha] = branch_name + return set(branches.values()) + + def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=False): + """ + Initializes the object and clones the git repository if it is not already present + """ + + # This allows us to set this one time and then keep track of the user's choice + ModulesRepo.no_pull_global |= no_pull + + # Check if the remote seems to be well formed + if remote_url is None: + remote_url = NF_CORE_MODULES_REMOTE + + self.remote_url = remote_url + + self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) + + self.setup_local_repo(remote_url, branch, hide_progress) + + config_fn, repo_config = load_tools_config(self.local_repo_dir) + try: + self.repo_path = repo_config["org_path"] + except KeyError: + raise UserWarning(f"'org_path' key not present in {config_fn.name}") + + # Verify that the repo seems to be correctly configured + if self.repo_path != NF_CORE_MODULES_NAME or self.branch: + self.verify_branch() + + # Convenience variable + self.modules_dir = os.path.join(self.local_repo_dir, "modules", self.repo_path) + self.subworkflows_dir = os.path.join(self.local_repo_dir, "subworkflows", self.repo_path) + + self.avail_module_names = None + + def verify_sha(self, prompt, sha): + """ + Verify that 'sha' and 'prompt' arguments are not provided together. + Verify that the provided SHA exists in the repo. + + Arguments: + prompt (bool): prompt asking for SHA + sha (str): provided sha + """ + if prompt and sha is not None: + log.error("Cannot use '--sha' and '--prompt' at the same time!") + return False + + if sha: + if not self.sha_exists_on_branch(sha): + log.error(f"Commit SHA '{sha}' doesn't exist in '{self.remote_url}'") + return False + + return True + + def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): + """ + Sets up the local git repository. If the repository has been cloned previously, it + returns a git.Repo object of that clone. Otherwise it tries to clone the repository from + the provided remote URL and returns a git.Repo of the new clone. + + Args: + remote (str): git url of remote + branch (str): name of branch to use + Sets self.repo + """ + self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname) + try: + if not os.path.exists(self.local_repo_dir): + try: + pbar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[bold yellow]{task.fields[state]}", + transient=True, + disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, + ) + with pbar: + self.repo = git.Repo.clone_from( + remote, + self.local_repo_dir, + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Cloning"), + ) + ModulesRepo.update_local_repo_status(self.fullname, True) + except GitCommandError: + raise LookupError(f"Failed to clone from the remote: `{remote}`") + # Verify that the requested branch exists by checking it out + self.setup_branch(branch) + else: + self.repo = git.Repo(self.local_repo_dir) + + if ModulesRepo.no_pull_global: + ModulesRepo.update_local_repo_status(self.fullname, True) + # If the repo is already cloned, fetch the latest changes from the remote + if not ModulesRepo.local_repo_synced(self.fullname): + pbar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[bold yellow]{task.fields[state]}", + transient=True, + disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, + ) + with pbar: + self.repo.remotes.origin.fetch( + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") + ) + ModulesRepo.update_local_repo_status(self.fullname, True) + + # Before verifying the branch, fetch the changes + # Verify that the requested branch exists by checking it out + self.setup_branch(branch) + + # Now merge the changes + tracking_branch = self.repo.active_branch.tracking_branch() + if tracking_branch is None: + raise LookupError(f"There is no remote tracking branch '{self.branch}' in '{self.remote_url}'") + self.repo.git.merge(tracking_branch.name) + except (GitCommandError, InvalidGitRepositoryError) as e: + log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") + if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): + log.info(f"Removing '{self.local_repo_dir}'") + shutil.rmtree(self.local_repo_dir) + self.setup_local_repo(remote, branch, hide_progress) + else: + raise LookupError("Exiting due to error with local modules git repo") + + def setup_branch(self, branch): + """ + Verify that we have a branch and otherwise use the default one. + The branch is then checked out to verify that it exists in the repo. + + Args: + branch (str): Name of branch + """ + if branch is None: + # Don't bother fetching default branch if we're using nf-core + if self.remote_url == NF_CORE_MODULES_REMOTE: + self.branch = "master" + else: + self.branch = self.get_default_branch() + else: + self.branch = branch + + # Verify that the branch exists by checking it out + self.branch_exists() + + def get_default_branch(self): + """ + Gets the default branch for the repo (the branch origin/HEAD is pointing to) + """ + origin_head = next(ref for ref in self.repo.refs if ref.name == "origin/HEAD") + _, branch = origin_head.ref.name.split("/") + return branch + + def branch_exists(self): + """ + Verifies that the branch exists in the repository by trying to check it out + """ + try: + self.checkout_branch() + except GitCommandError: + raise LookupError(f"Branch '{self.branch}' not found in '{self.remote_url}'") + + def verify_branch(self): + """ + Verifies the active branch conforms do the correct directory structure + """ + dir_names = os.listdir(self.local_repo_dir) + if "modules" not in dir_names: + err_str = f"Repository '{self.remote_url}' ({self.branch}) does not contain the 'modules/' directory" + if "software" in dir_names: + err_str += ( + ".\nAs of nf-core/tools version 2.0, the 'software/' directory should be renamed to 'modules/'" + ) + raise LookupError(err_str) + + def checkout_branch(self): + """ + Checks out the specified branch of the repository + """ + self.repo.git.checkout(self.branch) + + def checkout(self, commit): + """ + Checks out the repository at the requested commit + + Args: + commit (str): Git SHA of the commit + """ + self.repo.git.checkout(commit) + + def component_exists(self, component_name, component_type, checkout=True, commit=None): + """ + Check if a module/subworkflow exists in the branch of the repo + + Args: + component_name (str): The name of the module/subworkflow + + Returns: + (bool): Whether the module/subworkflow exists in this branch of the repository + """ + return component_name in self.get_avail_components(component_type, checkout=checkout, commit=commit) + + def get_component_dir(self, component_name, component_type): + """ + Returns the file path of a module/subworkflow directory in the repo. + Does not verify that the path exists. + Args: + component_name (str): The name of the module/subworkflow + + Returns: + component_path (str): The path of the module/subworkflow in the local copy of the repository + """ + if component_type == "modules": + return os.path.join(self.modules_dir, component_name) + elif component_type == "subworkflows": + return os.path.join(self.subworkflows_dir, component_name) + + def install_component(self, component_name, install_dir, commit, component_type): + """ + Install the module/subworkflow files into a pipeline at the given commit + + Args: + component_name (str): The name of the module/subworkflow + install_dir (str): The path where the module/subworkflow should be installed + commit (str): The git SHA for the version of the module/subworkflow to be installed + + Returns: + (bool): Whether the operation was successful or not + """ + # Check out the repository at the requested ref + try: + self.checkout(commit) + except git.GitCommandError: + return False + + # Check if the module/subworkflow exists in the branch + if not self.component_exists(component_name, component_type, checkout=False): + log.error( + f"The requested {component_type[:-1]} does not exists in the branch '{self.branch}' of {self.remote_url}'" + ) + return False + + # Copy the files from the repo to the install folder + shutil.copytree(self.get_component_dir(component_name, component_type), Path(install_dir, component_name)) + + # Switch back to the tip of the branch + self.checkout_branch() + return True + + def module_files_identical(self, module_name, base_path, commit): + """ + Checks whether the module files in a pipeline are identical to the ones in the remote + Args: + module_name (str): The name of the module + base_path (str): The path to the module in the pipeline + + Returns: + (bool): Whether the pipeline files are identical to the repo files + """ + if commit is None: + self.checkout_branch() + else: + self.checkout(commit) + module_files = ["main.nf", "meta.yml"] + files_identical = {file: True for file in module_files} + module_dir = self.get_component_dir(module_name, "modules") + for file in module_files: + try: + files_identical[file] = filecmp.cmp(os.path.join(module_dir, file), os.path.join(base_path, file)) + except FileNotFoundError: + log.debug(f"Could not open file: {os.path.join(module_dir, file)}") + continue + self.checkout_branch() + return files_identical + + def get_component_git_log(self, component_name, component_type, depth=None): + """ + Fetches the commit history the of requested module/subworkflow since a given date. The default value is + not arbitrary - it is the last time the structure of the nf-core/modules repository was had an + update breaking backwards compatibility. + Args: + component_name (str): Name of module/subworkflow + modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question + + Returns: + ( dict ): Iterator of commit SHAs and associated (truncated) message + """ + self.checkout_branch() + component_path = os.path.join(component_type, self.repo_path, component_name) + commits_new = self.repo.iter_commits(max_count=depth, paths=component_path) + commits_new = [ + {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_new + ] + commits_old = [] + if component_type == "modules": + # Grab commits also from previous modules structure + component_path = os.path.join("modules", component_name) + commits_old = self.repo.iter_commits(max_count=depth, paths=component_path) + commits_old = [ + {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_old + ] + commits = iter(commits_new + commits_old) + return commits + + def get_latest_component_version(self, component_name, component_type): + """ + Returns the latest commit in the repository + """ + return list(self.get_component_git_log(component_name, component_type, depth=1))[0]["git_sha"] + + def sha_exists_on_branch(self, sha): + """ + Verifies that a given commit sha exists on the branch + """ + self.checkout_branch() + return sha in (commit.hexsha for commit in self.repo.iter_commits()) + + def get_commit_info(self, sha): + """ + Fetches metadata about the commit (dates, message, etc.) + Args: + commit_sha (str): The SHA of the requested commit + Returns: + message (str): The commit message for the requested commit + date (str): The commit date for the requested commit + Raises: + LookupError: If the search for the commit fails + """ + self.checkout_branch() + for commit in self.repo.iter_commits(): + if commit.hexsha == sha: + message = commit.message.partition("\n")[0] + date_obj = commit.committed_datetime + date = str(date_obj.date()) + return message, date + raise LookupError(f"Commit '{sha}' not found in the '{self.remote_url}'") + + def get_avail_components(self, component_type, checkout=True, commit=None): + """ + Gets the names of the modules/subworkflows in the repository. They are detected by + checking which directories have a 'main.nf' file + + Returns: + ([ str ]): The module/subworkflow names + """ + if checkout: + self.checkout_branch() + if commit is not None: + self.checkout(commit) + # Get directory + if component_type == "modules": + directory = self.modules_dir + elif component_type == "subworkflows": + directory = self.subworkflows_dir + # Module/Subworkflow directories are characterized by having a 'main.nf' file + avail_component_names = [ + os.path.relpath(dirpath, start=directory) + for dirpath, _, file_names in os.walk(directory) + if "main.nf" in file_names + ] + return avail_component_names + + def get_meta_yml(self, component_type, module_name): + """ + Returns the contents of the 'meta.yml' file of a module + + Args: + module_name (str): The name of the module + + Returns: + (str): The contents of the file in text format + """ + self.checkout_branch() + if component_type == "modules": + path = Path(self.modules_dir, module_name, "meta.yml") + elif component_type == "subworkflows": + path = Path(self.subworkflows_dir, module_name, "meta.yml") + else: + raise ValueError(f"Invalid component type: {component_type}") + if not path.exists(): + return None + with open(path) as fh: + contents = fh.read() + return contents From 33d03812201ae59047fa8cf48ed06a5e1d055846 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 21 Feb 2023 15:11:01 +0100 Subject: [PATCH 057/249] Split history ./modules/modules_repo.py to synced_repo.py --- nf_core/{temp => modules/modules_repo.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nf_core/{temp => modules/modules_repo.py} (100%) diff --git a/nf_core/temp b/nf_core/modules/modules_repo.py similarity index 100% rename from nf_core/temp rename to nf_core/modules/modules_repo.py From caef187c811f5b28e5b326adb59c012fad726ae5 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 21 Feb 2023 15:53:19 +0100 Subject: [PATCH 058/249] Duplication of ModulesRepo to SyncedRepo done. --- nf_core/download.py | 15 ++------------- nf_core/modules/modules_repo.py | 3 ++- nf_core/synced_repo.py | 31 +++++++++++-------------------- 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 74b8abf3d8..e92e50164f 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -24,7 +24,7 @@ import nf_core import nf_core.list import nf_core.utils -from nf_core.modules import ModulesRepo # to create subclass WorkflowRepo +from nf_core.synced_repo import SyncedRepo # to create subclass WorkflowRepo log = logging.getLogger(__name__) stderr = rich.console.Console( @@ -826,7 +826,7 @@ def compress_download(self): log.info(f"MD5 checksum for '{self.output_filename}': [blue]{nf_core.utils.file_md5(self.output_filename)}[/]") -class WorkflowRepo(ModulesRepo): +class WorkflowRepo(SyncedRepo): """ An object to store details about a locally cached workflow repository. @@ -851,14 +851,3 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) self.setup_local_repo(remote_url, branch, hide_progress, in_cache=in_cache) - - @property - def active_branch(self): - """ - In ModuleRepo.setup_local_repo(), self.repo.active_branch.tracking_branch() is called in line 227. - For a WorkflowRepo, this raises a TypeError ``HEAD is a detached symbolic reference as it points to {commit hash}`` - - This property shadows the call and seemed the cleanest solution to prevent excessive code duplication. - Otherwise, I would have needed to define a setup_local_repo() method for the WorkflowRepo class. - """ - pass # TODO diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 23f62bdee2..5e4d80be16 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -12,6 +12,7 @@ import nf_core.modules.modules_json import nf_core.modules.modules_utils from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config +from nf_core.synced_repo import SyncedRepo log = logging.getLogger(__name__) @@ -58,7 +59,7 @@ def update(self, op_code, cur_count, max_count=None, message=""): ) -class ModulesRepo: +class ModulesRepo(SyncedRepo): """ An object to store details about the repository being used for modules. diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 23f62bdee2..89d2f894b1 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -9,8 +9,6 @@ import rich.progress from git.exc import GitCommandError, InvalidGitRepositoryError -import nf_core.modules.modules_json -import nf_core.modules.modules_utils from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config log = logging.getLogger(__name__) @@ -58,16 +56,9 @@ def update(self, op_code, cur_count, max_count=None, message=""): ) -class ModulesRepo: +class SyncedRepo: """ - An object to store details about the repository being used for modules. - - Used by the `nf-core modules` top-level command with -r and -b flags, - so that this can be used in the same way by all sub-commands. - - We keep track of the pull-status of the different installed repos in - the static variable local_repo_status. This is so we don't need to - pull a remote several times in one command. + An object to store details about a locally cached code repository. """ local_repo_statuses = {} @@ -78,14 +69,14 @@ def local_repo_synced(repo_name): """ Checks whether a local repo has been cloned/pull in the current session """ - return ModulesRepo.local_repo_statuses.get(repo_name, False) + return SyncedRepo.local_repo_statuses.get(repo_name, False) @staticmethod def update_local_repo_status(repo_name, up_to_date): """ Updates the clone/pull status of a local repo """ - ModulesRepo.local_repo_statuses[repo_name] = up_to_date + SyncedRepo.local_repo_statuses[repo_name] = up_to_date @staticmethod def get_remote_branches(remote_url): @@ -118,7 +109,7 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa """ # This allows us to set this one time and then keep track of the user's choice - ModulesRepo.no_pull_global |= no_pull + SyncedRepo.no_pull_global |= no_pull # Check if the remote seems to be well formed if remote_url is None: @@ -194,7 +185,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): self.local_repo_dir, progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Cloning"), ) - ModulesRepo.update_local_repo_status(self.fullname, True) + SyncedRepo.update_local_repo_status(self.fullname, True) except GitCommandError: raise LookupError(f"Failed to clone from the remote: `{remote}`") # Verify that the requested branch exists by checking it out @@ -202,10 +193,10 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): else: self.repo = git.Repo(self.local_repo_dir) - if ModulesRepo.no_pull_global: - ModulesRepo.update_local_repo_status(self.fullname, True) + if SyncedRepo.no_pull_global: + SyncedRepo.update_local_repo_status(self.fullname, True) # If the repo is already cloned, fetch the latest changes from the remote - if not ModulesRepo.local_repo_synced(self.fullname): + if not SyncedRepo.local_repo_synced(self.fullname): pbar = rich.progress.Progress( "[bold blue]{task.description}", rich.progress.BarColumn(bar_width=None), @@ -217,7 +208,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): self.repo.remotes.origin.fetch( progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") ) - ModulesRepo.update_local_repo_status(self.fullname, True) + SyncedRepo.update_local_repo_status(self.fullname, True) # Before verifying the branch, fetch the changes # Verify that the requested branch exists by checking it out @@ -394,7 +385,7 @@ def get_component_git_log(self, component_name, component_type, depth=None): update breaking backwards compatibility. Args: component_name (str): Name of module/subworkflow - modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question + modules_repo (SyncedRepo): A SyncedRepo object configured for the repository in question Returns: ( dict ): Iterator of commit SHAs and associated (truncated) message From 2367ae736e1e7de54f636f576200c5933a425606 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 21 Feb 2023 15:59:03 +0100 Subject: [PATCH 059/249] Strip ModulesRepo class of the methods moved to new superclass. --- nf_core/modules/modules_repo.py | 358 +------------------------------- 1 file changed, 1 insertion(+), 357 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 5e4d80be16..20d581af84 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -12,7 +12,7 @@ import nf_core.modules.modules_json import nf_core.modules.modules_utils from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config -from nf_core.synced_repo import SyncedRepo +from nf_core.synced_repo import RemoteProgressbar, SyncedRepo log = logging.getLogger(__name__) @@ -22,43 +22,6 @@ NF_CORE_MODULES_DEFAULT_BRANCH = "master" -class RemoteProgressbar(git.RemoteProgress): - """ - An object to create a progressbar for when doing an operation with the remote. - Note that an initialized rich Progress (progress bar) object must be past - during initialization. - """ - - def __init__(self, progress_bar, repo_name, remote_url, operation): - """ - Initializes the object and adds a task to the progressbar passed as 'progress_bar' - - Args: - progress_bar (rich.progress.Progress): A rich progress bar object - repo_name (str): Name of the repository the operation is performed on - remote_url (str): Git URL of the repository the operation is performed on - operation (str): The operation performed on the repository, i.e. 'Pulling', 'Cloning' etc. - """ - super().__init__() - self.progress_bar = progress_bar - self.tid = self.progress_bar.add_task( - f"{operation} from [bold green]'{repo_name}'[/bold green] ([link={remote_url}]{remote_url}[/link])", - start=False, - state="Waiting for response", - ) - - def update(self, op_code, cur_count, max_count=None, message=""): - """ - Overrides git.RemoteProgress.update. - Called every time there is a change in the remote operation - """ - if not self.progress_bar.tasks[self.tid].started: - self.progress_bar.start_task(self.tid) - self.progress_bar.update( - self.tid, total=max_count, completed=cur_count, state=f"{cur_count / max_count * 100:.1f}%" - ) - - class ModulesRepo(SyncedRepo): """ An object to store details about the repository being used for modules. @@ -74,45 +37,6 @@ class ModulesRepo(SyncedRepo): local_repo_statuses = {} no_pull_global = False - @staticmethod - def local_repo_synced(repo_name): - """ - Checks whether a local repo has been cloned/pull in the current session - """ - return ModulesRepo.local_repo_statuses.get(repo_name, False) - - @staticmethod - def update_local_repo_status(repo_name, up_to_date): - """ - Updates the clone/pull status of a local repo - """ - ModulesRepo.local_repo_statuses[repo_name] = up_to_date - - @staticmethod - def get_remote_branches(remote_url): - """ - Get all branches from a remote repository - - Args: - remote_url (str): The git url to the remote repository - - Returns: - (set[str]): All branches found in the remote - """ - try: - unparsed_branches = git.Git().ls_remote(remote_url) - except git.GitCommandError: - raise LookupError(f"Was unable to fetch branches from '{remote_url}'") - else: - branches = {} - for branch_info in unparsed_branches.split("\n"): - sha, name = branch_info.split("\t") - if name != "HEAD": - # The remote branches are shown as 'ref/head/branch' - branch_name = Path(name).stem - branches[sha] = branch_name - return set(branches.values()) - def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=False): """ Initializes the object and clones the git repository if it is not already present @@ -147,26 +71,6 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.avail_module_names = None - def verify_sha(self, prompt, sha): - """ - Verify that 'sha' and 'prompt' arguments are not provided together. - Verify that the provided SHA exists in the repo. - - Arguments: - prompt (bool): prompt asking for SHA - sha (str): provided sha - """ - if prompt and sha is not None: - log.error("Cannot use '--sha' and '--prompt' at the same time!") - return False - - if sha: - if not self.sha_exists_on_branch(sha): - log.error(f"Commit SHA '{sha}' doesn't exist in '{self.remote_url}'") - return False - - return True - def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): """ Sets up the local git repository. If the repository has been cloned previously, it @@ -237,263 +141,3 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): self.setup_local_repo(remote, branch, hide_progress) else: raise LookupError("Exiting due to error with local modules git repo") - - def setup_branch(self, branch): - """ - Verify that we have a branch and otherwise use the default one. - The branch is then checked out to verify that it exists in the repo. - - Args: - branch (str): Name of branch - """ - if branch is None: - # Don't bother fetching default branch if we're using nf-core - if self.remote_url == NF_CORE_MODULES_REMOTE: - self.branch = "master" - else: - self.branch = self.get_default_branch() - else: - self.branch = branch - - # Verify that the branch exists by checking it out - self.branch_exists() - - def get_default_branch(self): - """ - Gets the default branch for the repo (the branch origin/HEAD is pointing to) - """ - origin_head = next(ref for ref in self.repo.refs if ref.name == "origin/HEAD") - _, branch = origin_head.ref.name.split("/") - return branch - - def branch_exists(self): - """ - Verifies that the branch exists in the repository by trying to check it out - """ - try: - self.checkout_branch() - except GitCommandError: - raise LookupError(f"Branch '{self.branch}' not found in '{self.remote_url}'") - - def verify_branch(self): - """ - Verifies the active branch conforms do the correct directory structure - """ - dir_names = os.listdir(self.local_repo_dir) - if "modules" not in dir_names: - err_str = f"Repository '{self.remote_url}' ({self.branch}) does not contain the 'modules/' directory" - if "software" in dir_names: - err_str += ( - ".\nAs of nf-core/tools version 2.0, the 'software/' directory should be renamed to 'modules/'" - ) - raise LookupError(err_str) - - def checkout_branch(self): - """ - Checks out the specified branch of the repository - """ - self.repo.git.checkout(self.branch) - - def checkout(self, commit): - """ - Checks out the repository at the requested commit - - Args: - commit (str): Git SHA of the commit - """ - self.repo.git.checkout(commit) - - def component_exists(self, component_name, component_type, checkout=True, commit=None): - """ - Check if a module/subworkflow exists in the branch of the repo - - Args: - component_name (str): The name of the module/subworkflow - - Returns: - (bool): Whether the module/subworkflow exists in this branch of the repository - """ - return component_name in self.get_avail_components(component_type, checkout=checkout, commit=commit) - - def get_component_dir(self, component_name, component_type): - """ - Returns the file path of a module/subworkflow directory in the repo. - Does not verify that the path exists. - Args: - component_name (str): The name of the module/subworkflow - - Returns: - component_path (str): The path of the module/subworkflow in the local copy of the repository - """ - if component_type == "modules": - return os.path.join(self.modules_dir, component_name) - elif component_type == "subworkflows": - return os.path.join(self.subworkflows_dir, component_name) - - def install_component(self, component_name, install_dir, commit, component_type): - """ - Install the module/subworkflow files into a pipeline at the given commit - - Args: - component_name (str): The name of the module/subworkflow - install_dir (str): The path where the module/subworkflow should be installed - commit (str): The git SHA for the version of the module/subworkflow to be installed - - Returns: - (bool): Whether the operation was successful or not - """ - # Check out the repository at the requested ref - try: - self.checkout(commit) - except git.GitCommandError: - return False - - # Check if the module/subworkflow exists in the branch - if not self.component_exists(component_name, component_type, checkout=False): - log.error( - f"The requested {component_type[:-1]} does not exists in the branch '{self.branch}' of {self.remote_url}'" - ) - return False - - # Copy the files from the repo to the install folder - shutil.copytree(self.get_component_dir(component_name, component_type), Path(install_dir, component_name)) - - # Switch back to the tip of the branch - self.checkout_branch() - return True - - def module_files_identical(self, module_name, base_path, commit): - """ - Checks whether the module files in a pipeline are identical to the ones in the remote - Args: - module_name (str): The name of the module - base_path (str): The path to the module in the pipeline - - Returns: - (bool): Whether the pipeline files are identical to the repo files - """ - if commit is None: - self.checkout_branch() - else: - self.checkout(commit) - module_files = ["main.nf", "meta.yml"] - files_identical = {file: True for file in module_files} - module_dir = self.get_component_dir(module_name, "modules") - for file in module_files: - try: - files_identical[file] = filecmp.cmp(os.path.join(module_dir, file), os.path.join(base_path, file)) - except FileNotFoundError: - log.debug(f"Could not open file: {os.path.join(module_dir, file)}") - continue - self.checkout_branch() - return files_identical - - def get_component_git_log(self, component_name, component_type, depth=None): - """ - Fetches the commit history the of requested module/subworkflow since a given date. The default value is - not arbitrary - it is the last time the structure of the nf-core/modules repository was had an - update breaking backwards compatibility. - Args: - component_name (str): Name of module/subworkflow - modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question - - Returns: - ( dict ): Iterator of commit SHAs and associated (truncated) message - """ - self.checkout_branch() - component_path = os.path.join(component_type, self.repo_path, component_name) - commits_new = self.repo.iter_commits(max_count=depth, paths=component_path) - commits_new = [ - {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_new - ] - commits_old = [] - if component_type == "modules": - # Grab commits also from previous modules structure - component_path = os.path.join("modules", component_name) - commits_old = self.repo.iter_commits(max_count=depth, paths=component_path) - commits_old = [ - {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_old - ] - commits = iter(commits_new + commits_old) - return commits - - def get_latest_component_version(self, component_name, component_type): - """ - Returns the latest commit in the repository - """ - return list(self.get_component_git_log(component_name, component_type, depth=1))[0]["git_sha"] - - def sha_exists_on_branch(self, sha): - """ - Verifies that a given commit sha exists on the branch - """ - self.checkout_branch() - return sha in (commit.hexsha for commit in self.repo.iter_commits()) - - def get_commit_info(self, sha): - """ - Fetches metadata about the commit (dates, message, etc.) - Args: - commit_sha (str): The SHA of the requested commit - Returns: - message (str): The commit message for the requested commit - date (str): The commit date for the requested commit - Raises: - LookupError: If the search for the commit fails - """ - self.checkout_branch() - for commit in self.repo.iter_commits(): - if commit.hexsha == sha: - message = commit.message.partition("\n")[0] - date_obj = commit.committed_datetime - date = str(date_obj.date()) - return message, date - raise LookupError(f"Commit '{sha}' not found in the '{self.remote_url}'") - - def get_avail_components(self, component_type, checkout=True, commit=None): - """ - Gets the names of the modules/subworkflows in the repository. They are detected by - checking which directories have a 'main.nf' file - - Returns: - ([ str ]): The module/subworkflow names - """ - if checkout: - self.checkout_branch() - if commit is not None: - self.checkout(commit) - # Get directory - if component_type == "modules": - directory = self.modules_dir - elif component_type == "subworkflows": - directory = self.subworkflows_dir - # Module/Subworkflow directories are characterized by having a 'main.nf' file - avail_component_names = [ - os.path.relpath(dirpath, start=directory) - for dirpath, _, file_names in os.walk(directory) - if "main.nf" in file_names - ] - return avail_component_names - - def get_meta_yml(self, component_type, module_name): - """ - Returns the contents of the 'meta.yml' file of a module - - Args: - module_name (str): The name of the module - - Returns: - (str): The contents of the file in text format - """ - self.checkout_branch() - if component_type == "modules": - path = Path(self.modules_dir, module_name, "meta.yml") - elif component_type == "subworkflows": - path = Path(self.subworkflows_dir, module_name, "meta.yml") - else: - raise ValueError(f"Invalid component type: {component_type}") - if not path.exists(): - return None - with open(path) as fh: - contents = fh.read() - return contents From f5f0df2327fb524c5b2f519a52b5ef52e28a26c1 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 21 Feb 2023 17:18:38 +0100 Subject: [PATCH 060/249] Rebase to current dev. --- nf_core/synced_repo.py | 75 ++---------------------------------------- 1 file changed, 2 insertions(+), 73 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 89d2f894b1..715f6d77bb 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -7,9 +7,9 @@ import git import rich import rich.progress -from git.exc import GitCommandError, InvalidGitRepositoryError +from git.exc import GitCommandError -from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config +from nf_core.utils import load_tools_config log = logging.getLogger(__name__) @@ -157,77 +157,6 @@ def verify_sha(self, prompt, sha): return True - def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): - """ - Sets up the local git repository. If the repository has been cloned previously, it - returns a git.Repo object of that clone. Otherwise it tries to clone the repository from - the provided remote URL and returns a git.Repo of the new clone. - - Args: - remote (str): git url of remote - branch (str): name of branch to use - Sets self.repo - """ - self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname) - try: - if not os.path.exists(self.local_repo_dir): - try: - pbar = rich.progress.Progress( - "[bold blue]{task.description}", - rich.progress.BarColumn(bar_width=None), - "[bold yellow]{task.fields[state]}", - transient=True, - disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, - ) - with pbar: - self.repo = git.Repo.clone_from( - remote, - self.local_repo_dir, - progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Cloning"), - ) - SyncedRepo.update_local_repo_status(self.fullname, True) - except GitCommandError: - raise LookupError(f"Failed to clone from the remote: `{remote}`") - # Verify that the requested branch exists by checking it out - self.setup_branch(branch) - else: - self.repo = git.Repo(self.local_repo_dir) - - if SyncedRepo.no_pull_global: - SyncedRepo.update_local_repo_status(self.fullname, True) - # If the repo is already cloned, fetch the latest changes from the remote - if not SyncedRepo.local_repo_synced(self.fullname): - pbar = rich.progress.Progress( - "[bold blue]{task.description}", - rich.progress.BarColumn(bar_width=None), - "[bold yellow]{task.fields[state]}", - transient=True, - disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, - ) - with pbar: - self.repo.remotes.origin.fetch( - progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") - ) - SyncedRepo.update_local_repo_status(self.fullname, True) - - # Before verifying the branch, fetch the changes - # Verify that the requested branch exists by checking it out - self.setup_branch(branch) - - # Now merge the changes - tracking_branch = self.repo.active_branch.tracking_branch() - if tracking_branch is None: - raise LookupError(f"There is no remote tracking branch '{self.branch}' in '{self.remote_url}'") - self.repo.git.merge(tracking_branch.name) - except (GitCommandError, InvalidGitRepositoryError) as e: - log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") - if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): - log.info(f"Removing '{self.local_repo_dir}'") - shutil.rmtree(self.local_repo_dir) - self.setup_local_repo(remote, branch, hide_progress) - else: - raise LookupError("Exiting due to error with local modules git repo") - def setup_branch(self, branch): """ Verify that we have a branch and otherwise use the default one. From f852159b7d38e863d419aaa2e21a04724c8cb885 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 21 Feb 2023 21:40:40 +0100 Subject: [PATCH 061/249] Local caching of the repo works now. --- nf_core/download.py | 99 ++++++++++++++++++++++++++++++++++++------ nf_core/synced_repo.py | 2 +- 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index e92e50164f..67c987bb44 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -3,7 +3,8 @@ from __future__ import print_function import concurrent.futures -from git import Repo +import git +from git.exc import GitCommandError, InvalidGitRepositoryError import io import logging import os @@ -24,7 +25,8 @@ import nf_core import nf_core.list import nf_core.utils -from nf_core.synced_repo import SyncedRepo # to create subclass WorkflowRepo +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR +from nf_core.synced_repo import RemoteProgressbar, SyncedRepo log = logging.getLogger(__name__) stderr = rich.console.Console( @@ -127,7 +129,8 @@ def download_workflow(self): self.prompt_container_download() self.prompt_use_singularity_cachedir() self.prompt_singularity_cachedir_only() - self.prompt_compression_type() + if not self.tower: + self.prompt_compression_type() except AssertionError as e: log.critical(e) sys.exit(1) @@ -207,10 +210,10 @@ def download_workflow_tower(self): """Create a bare-cloned git repository of the workflow that includes the configurations, such it can be launched with `tw launch` as file:/ pipeline""" log.info("Collecting workflow from GitHub") - self.workflow_repo = WorkflowRepo(remote_url=f"git@github.com:{self.pipeline}.git", branch=self.revision) - import pbb + self.workflow_repo = WorkflowRepo( + remote_url=f"git@github.com:{self.pipeline}.git", revision=self.revision, commit=self.wf_sha + ) - pdb.set_trace() log.info("Downloading centralised configs from GitHub") def prompt_pipeline_name(self): @@ -625,12 +628,12 @@ def singularity_image_filenames(self, container): """Check Singularity cache for image, copy to destination folder if found. Args: - container (str): A pipeline's container name. Can be direct download URL - or a Docker Hub repository ID. + container (str): A pipeline's container name. Can be direct download URL + or a Docker Hub repository ID. Returns: - results (bool, str): Returns True if we have the image in the target location. - Returns a download path if not. + results (bool, str): Returns True if we have the image in the target location. + Returns a download path if not. """ # Generate file paths @@ -836,18 +839,86 @@ class WorkflowRepo(SyncedRepo): """ - def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=False, in_cache=True): + def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=False, in_cache=True): """ Initializes the object and clones the workflows git repository if it is not already present Args: - remote_url (str, optional): The URL of the remote repository. Defaults to None. - branch (str, optional): The branch to clone. Defaults to None. + remote_url (str): The URL of the remote repository. Defaults to None. + commit (str): The commit to clone. Defaults to None. no_pull (bool, optional): Whether to skip the pull step. Defaults to False. hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. """ self.remote_url = remote_url + self.revision = revision + self.commit = commit self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) - self.setup_local_repo(remote_url, branch, hide_progress, in_cache=in_cache) + self.setup_local_repo(remote_url, revision, commit, hide_progress, in_cache=in_cache) + + def setup_local_repo(self, remote, revision, commit, hide_progress=True, in_cache=True): + """ + Sets up the local git repository. If the repository has been cloned previously, it + returns a git.Repo object of that clone. Otherwise it tries to clone the repository from + the provided remote URL and returns a git.Repo of the new clone. + + Args: + remote (str): git url of remote + branch (str): name of branch to use + hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. + in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. + Sets self.repo + """ + + self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname) + try: + if not os.path.exists(self.local_repo_dir): + try: + pbar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[bold yellow]{task.fields[state]}", + transient=True, + disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, + ) + with pbar: + self.repo = git.Repo.clone_from( + remote, + self.local_repo_dir, + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Cloning"), + ) + super().update_local_repo_status(self.fullname, True) + except GitCommandError: + raise LookupError(f"Failed to clone from the remote: `{remote}`") + else: + self.repo = git.Repo(self.local_repo_dir) + + if super().no_pull_global: + super().update_local_repo_status(self.fullname, True) + # If the repo is already cloned, fetch the latest changes from the remote + if not super().local_repo_synced(self.fullname): + pbar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[bold yellow]{task.fields[state]}", + transient=True, + disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, + ) + with pbar: + self.repo.remotes.origin.fetch( + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") + ) + super().update_local_repo_status(self.fullname, True) + + except (GitCommandError, InvalidGitRepositoryError) as e: + log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") + if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): + log.info(f"Removing '{self.local_repo_dir}'") + shutil.rmtree(self.local_repo_dir) + self.setup_local_repo(remote, revision, commit, hide_progress) + else: + raise LookupError("Exiting due to error with local modules git repo") + + finally: + self.repo.git.checkout(commit) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 715f6d77bb..f04ef8e0c7 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -9,7 +9,7 @@ import rich.progress from git.exc import GitCommandError -from nf_core.utils import load_tools_config +from nf_core.utils import load_tools_config log = logging.getLogger(__name__) From af4754e05d02dc1239861fb8819ab8c0c6f61f9f Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 22 Feb 2023 20:13:07 +0100 Subject: [PATCH 062/249] Started implementing the config download. --- nf_core/download.py | 70 +++++++++++++++++++++++++++++++++++++----- nf_core/synced_repo.py | 2 +- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 67c987bb44..0d33b601f0 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -100,6 +100,7 @@ def __init__( self.compress_type = compress_type self.force = force self.tower = tower + self.include_config self.container = container self.singularity_cache_only = singularity_cache_only self.parallel_downloads = parallel_downloads @@ -210,10 +211,13 @@ def download_workflow_tower(self): """Create a bare-cloned git repository of the workflow that includes the configurations, such it can be launched with `tw launch` as file:/ pipeline""" log.info("Collecting workflow from GitHub") + self.workflow_repo = WorkflowRepo( remote_url=f"git@github.com:{self.pipeline}.git", revision=self.revision, commit=self.wf_sha ) + import pdb + pdb.set_trace() log.info("Downloading centralised configs from GitHub") def prompt_pipeline_name(self): @@ -374,6 +378,12 @@ def prompt_compression_type(self): if self.compress_type == "none": self.compress_type = None + def prompt_config_inclusion(self): + """Prompt for inclusion of institutional configurations""" + self.include_configs = questionary.confirm( + "Include the institutional configuration files into the download?" + ).ask() + def download_wf_files(self): """Downloads workflow files from GitHub to the :attr:`self.outdir`.""" log.debug(f"Downloading {self.wf_download_url}") @@ -853,11 +863,24 @@ def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=Fa self.remote_url = remote_url self.revision = revision self.commit = commit + self.hide_progress = hide_progress self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) - self.setup_local_repo(remote_url, revision, commit, hide_progress, in_cache=in_cache) + self.setup_local_repo(remote_url, commit, hide_progress, in_cache=in_cache) + + def __repr__(self): + """Called by print, creates representation of object""" + return f"" + + def retry_setup_local_repo(self): + if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): + log.info(f"Removing '{self.local_repo_dir}'") + shutil.rmtree(self.local_repo_dir) + self.setup_local_repo(self.remote, self.commit, self.hide_progress) + else: + raise LookupError("Exiting due to error with local modules git repo") - def setup_local_repo(self, remote, revision, commit, hide_progress=True, in_cache=True): + def setup_local_repo(self, remote, commit, hide_progress=False, in_cache=True): """ Sets up the local git repository. If the repository has been cloned previously, it returns a git.Repo object of that clone. Otherwise it tries to clone the repository from @@ -913,12 +936,43 @@ def setup_local_repo(self, remote, revision, commit, hide_progress=True, in_cach except (GitCommandError, InvalidGitRepositoryError) as e: log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") - if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): - log.info(f"Removing '{self.local_repo_dir}'") - shutil.rmtree(self.local_repo_dir) - self.setup_local_repo(remote, revision, commit, hide_progress) - else: - raise LookupError("Exiting due to error with local modules git repo") + self.retry_setup_local_repo() + finally: + self.repo.git.checkout(commit) + + def add_nfcore_configs(self, commit, hide_progress=False): + """ + Pulls the configuration profiles from the nf-core/config repository on GitHub. + + Args: + commit: The config version to pull + hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. + Sets self.repo + """ + try: + if os.path.exists(self.local_repo_dir): + try: + pbar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[bold yellow]{task.fields[state]}", + transient=True, + disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, + ) + with pbar: + self.configs = git.Submodule.add( + self.repo, + "nf-core configuration", + "./conf_institutional", + f"git@github.com:nf-core/configs.git", + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Adding configuration"), + ) + except GitCommandError: + raise LookupError(f"Failed to retrieve configuration: `{remote}`") + + except (GitCommandError, InvalidGitRepositoryError) as e: + log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") + self.retry_setup_local_repo() finally: self.repo.git.checkout(commit) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index f04ef8e0c7..4bbd4f8443 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -22,7 +22,7 @@ class RemoteProgressbar(git.RemoteProgress): """ An object to create a progressbar for when doing an operation with the remote. - Note that an initialized rich Progress (progress bar) object must be past + Note that an initialized rich Progress (progress bar) object must be passed during initialization. """ From 3bc97c553de3126dd930e3d6ef1882f25b70241c Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 24 Feb 2023 18:42:12 +0100 Subject: [PATCH 063/249] Started to implement the multiple revision selection for the Tower download. --- nf_core/download.py | 78 ++++++++++++++++++++++----------------------- nf_core/utils.py | 10 ++++-- 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 0d33b601f0..ff06faecdc 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -75,7 +75,7 @@ class DownloadWorkflow: Args: pipeline (str): A nf-core pipeline name. - revision (str): The workflow revision to download, like `1.0`. Defaults to None. + revision (List[str]): The workflow revision to download, like `1.0`. Defaults to None. singularity (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False. tower (bool): Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. outdir (str): Path to the local download directory. Defaults to None. @@ -94,20 +94,19 @@ def __init__( parallel_downloads=4, ): self.pipeline = pipeline - self.revision = revision + self.revision = [].extend(revision) if revision else [] self.outdir = outdir self.output_filename = None self.compress_type = compress_type self.force = force self.tower = tower - self.include_config self.container = container self.singularity_cache_only = singularity_cache_only self.parallel_downloads = parallel_downloads self.wf_revisions = {} self.wf_branches = {} - self.wf_sha = None + self.wf_sha = {} self.wf_download_url = None self.nf_config = {} self.containers = [] @@ -136,7 +135,7 @@ def download_workflow(self): log.critical(e) sys.exit(1) - summary_log = [f"Pipeline revision: '{self.revision}'", f"Pull containers: '{self.container}'"] + summary_log = [f"Pipeline revision: '{','.join(self.revision)}'", f"Pull containers: '{self.container}'"] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: summary_log.append(f"Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {os.environ['NXF_SINGULARITY_CACHEDIR']}") @@ -213,11 +212,10 @@ def download_workflow_tower(self): log.info("Collecting workflow from GitHub") self.workflow_repo = WorkflowRepo( - remote_url=f"git@github.com:{self.pipeline}.git", revision=self.revision, commit=self.wf_sha + remote_url=f"git@github.com:{self.pipeline}.git", + revision=self.revision[0] if self.revision else None, + commit=list(self.wf_sha.values())[0] if bool(self.wf_sha) else "", ) - import pdb - - pdb.set_trace() log.info("Downloading centralised configs from GitHub") def prompt_pipeline_name(self): @@ -230,39 +228,44 @@ def prompt_pipeline_name(self): def prompt_revision(self): """Prompt for pipeline revision / branch""" # Prompt user for revision tag if '--revision' was not set - if self.revision is None: - self.revision = nf_core.utils.prompt_pipeline_release_branch(self.wf_revisions, self.wf_branches) + # If --tower is specified, allow to select multiple revisions + + if not bool(self.revision): + self.revision.extend( + nf_core.utils.prompt_pipeline_release_branch(self.wf_revisions, self.wf_branches, multiple=self.tower) + ) def get_revision_hash(self): """Find specified revision / branch hash""" - # Branch - if self.revision in self.wf_branches.keys(): - self.wf_sha = self.wf_branches[self.revision] - - # Revision - else: - for r in self.wf_revisions: - if r["tag_name"] == self.revision: - self.wf_sha = r["tag_sha"] - break + for revision in self.revision: # revision is a list of strings, but may be of length 1 + # Branch + if revision in self.wf_branches.keys(): + self.wf_sha[revision].append(self.wf_branches[revision]) - # Can't find the revisions or branch - throw an error + # Revision else: - log.info( - "Available {} revisions: '{}'".format( - self.pipeline, "', '".join([r["tag_name"] for r in self.wf_revisions]) + for r in self.wf_revisions: + if r["tag_name"] == revision: + self.wf_sha[revision].append(r["tag_sha"]) + break + + # Can't find the revisions or branch - throw an error + else: + log.info( + "Available {} revisions: '{}'".format( + self.pipeline, "', '".join([r["tag_name"] for r in self.wf_revisions]) + ) ) - ) - log.info("Available {} branches: '{}'".format(self.pipeline, "', '".join(self.wf_branches.keys()))) - raise AssertionError(f"Not able to find revision / branch '{self.revision}' for {self.pipeline}") + log.info("Available {} branches: '{}'".format(self.pipeline, "', '".join(self.wf_branches.keys()))) + raise AssertionError(f"Not able to find revision / branch '{revision}' for {self.pipeline}") # Set the outdir if not self.outdir: - self.outdir = f"{self.pipeline.replace('/', '-').lower()}-{self.revision}" + self.outdir = f"{self.pipeline.replace('/', '-').lower()}-{self.revision[0] if self.revision else ''}" # Set the download URL and return - self.wf_download_url = f"https://github.com/{self.pipeline}/archive/{self.wf_sha}.zip" + self.wf_download_url = f"https://github.com/{self.pipeline}/archive/{list(self.wf_sha.values())[0] if bool(self.wf_sha) else ''}.zip" def prompt_container_download(self): """Prompt whether to download container images or not""" @@ -378,12 +381,6 @@ def prompt_compression_type(self): if self.compress_type == "none": self.compress_type = None - def prompt_config_inclusion(self): - """Prompt for inclusion of institutional configurations""" - self.include_configs = questionary.confirm( - "Include the institutional configuration files into the download?" - ).ask() - def download_wf_files(self): """Downloads workflow files from GitHub to the :attr:`self.outdir`.""" log.debug(f"Downloading {self.wf_download_url}") @@ -394,7 +391,7 @@ def download_wf_files(self): zipfile.extractall(self.outdir) # Rename the internal directory name to be more friendly - gh_name = f"{self.pipeline}-{self.wf_sha}".split("/")[-1] + gh_name = f"{self.pipeline}-{list(self.wf_sha.values())[0] if bool(self.wf_sha) else ''}".split("/")[-1] os.rename(os.path.join(self.outdir, gh_name), os.path.join(self.outdir, "workflow")) # Make downloaded files executable @@ -795,7 +792,7 @@ def singularity_pull_image(self, container, out_path, cache_path, progress): if lines: # something went wrong with the container retrieval if any("FATAL: " in line for line in lines): - log.info("Singularity container retrieval fialed with the following error:") + log.info("Singularity container retrieval failed with the following error:") log.info("".join(lines)) raise FileNotFoundError(f'The container "{container}" is unavailable.\n{"".join(lines)}') @@ -855,14 +852,15 @@ def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=Fa Args: remote_url (str): The URL of the remote repository. Defaults to None. + self.revision (list): The revision to use. A list of strings. commit (str): The commit to clone. Defaults to None. no_pull (bool, optional): Whether to skip the pull step. Defaults to False. hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. """ self.remote_url = remote_url - self.revision = revision - self.commit = commit + self.revision = [].extend(revision) if revision else [] + self.commit = [].extend(commit) if commit else [] self.hide_progress = hide_progress self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) diff --git a/nf_core/utils.py b/nf_core/utils.py index 3fa72d8b53..cef1cf3272 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -823,12 +823,13 @@ def prompt_remote_pipeline_name(wfs): raise AssertionError(f"Not able to find pipeline '{pipeline}'") -def prompt_pipeline_release_branch(wf_releases, wf_branches): +def prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False): """Prompt for pipeline release / branch Args: wf_releases (array): Array of repo releases as returned by the GitHub API wf_branches (array): Array of repo branches, as returned by the GitHub API + multiple (bool): Allow selection of multiple releases & branches (for Tower) Returns: choice (str): Selected release / branch name @@ -850,7 +851,12 @@ def prompt_pipeline_release_branch(wf_releases, wf_branches): if len(choices) == 0: return False - return questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask() + if multiple: + return questionary.checkbox( + "Select release / branch:", choices=choices, style=nfcore_question_style + ).unsafe_ask() + else: + return questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask() def get_repo_releases_branches(pipeline, wfs): From e17a8e9dba648575824c6714353f149aa213df4b Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 28 Feb 2023 15:19:53 +0100 Subject: [PATCH 064/249] Rewrite get_revision_hash() function to accomodate multiple revisions. --- nf_core/download.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index ff06faecdc..87a618cee0 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -241,13 +241,13 @@ def get_revision_hash(self): for revision in self.revision: # revision is a list of strings, but may be of length 1 # Branch if revision in self.wf_branches.keys(): - self.wf_sha[revision].append(self.wf_branches[revision]) + self.wf_sha = {**self.wf_sha, revision: self.wf_branches[revision]} # Revision else: for r in self.wf_revisions: if r["tag_name"] == revision: - self.wf_sha[revision].append(r["tag_sha"]) + self.wf_sha = {**self.wf_sha, revision: r["tag_sha"]} break # Can't find the revisions or branch - throw an error @@ -262,10 +262,13 @@ def get_revision_hash(self): # Set the outdir if not self.outdir: - self.outdir = f"{self.pipeline.replace('/', '-').lower()}-{self.revision[0] if self.revision else ''}" + self.outdir = ( + f"{self.pipeline.replace('/', '-').lower()}-{'_'.join(self.revision) if self.revision else ''}" + ) - # Set the download URL and return - self.wf_download_url = f"https://github.com/{self.pipeline}/archive/{list(self.wf_sha.values())[0] if bool(self.wf_sha) else ''}.zip" + if not self.tower and bool(self.wf_sha): + # Set the download URL and return - only applicable for classic downloads + self.wf_download_url = f"https://github.com/{self.pipeline}/archive/{list(self.wf_sha.values())[0]}.zip" def prompt_container_download(self): """Prompt whether to download container images or not""" From ecaabf8bd5e07a1e289f51086fc3ea6778f70f06 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 28 Feb 2023 15:40:02 +0100 Subject: [PATCH 065/249] The 2nd revivial of the config choice. Now available for archives with only one revision. --- nf_core/download.py | 55 ++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 87a618cee0..41b1fe1458 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -100,6 +100,7 @@ def __init__( self.compress_type = compress_type self.force = force self.tower = tower + self.include_configs = True self.container = container self.singularity_cache_only = singularity_cache_only self.parallel_downloads = parallel_downloads @@ -126,6 +127,9 @@ def download_workflow(self): ) self.prompt_revision() self.get_revision_hash() + # inclusion of configs is unsuitable for multi-revision repositories. + if len(self.revision) == 1: + self.prompt_config_inclusion() self.prompt_container_download() self.prompt_use_singularity_cachedir() self.prompt_singularity_cachedir_only() @@ -149,6 +153,10 @@ def download_workflow(self): else: summary_log.append(f"Output directory: '{self.outdir}'") + if len(self.revision) == 1: + # Only show entry, if option was prompted. + summary_log.append(f"Include default institutional configuration: '{self.include_configs}'") + summary_log.append(f"Enabled for seqeralabs® Nextflow Tower: '{self.tower}'") # Check that the outdir doesn't already exist @@ -183,14 +191,15 @@ def download_workflow_classic(self): self.download_wf_files() # Download the centralised configs - log.info("Downloading centralised configs from GitHub") - self.download_configs() - try: - self.wf_use_local_configs() - except FileNotFoundError as e: - log.error("Error editing pipeline config file to use local configs!") - log.critical(e) - sys.exit(1) + if self.include_configs: + log.info("Downloading centralised configs from GitHub") + self.download_configs() + try: + self.wf_use_local_configs() + except FileNotFoundError as e: + log.error("Error editing pipeline config file to use local configs!") + log.critical(e) + sys.exit(1) # Download the singularity images if self.container == "singularity": @@ -213,10 +222,12 @@ def download_workflow_tower(self): self.workflow_repo = WorkflowRepo( remote_url=f"git@github.com:{self.pipeline}.git", - revision=self.revision[0] if self.revision else None, - commit=list(self.wf_sha.values())[0] if bool(self.wf_sha) else "", + revision=self.revision if self.revision else None, + commit=self.wf_sha.values if bool(self.wf_sha) else None, ) - log.info("Downloading centralised configs from GitHub") + + if self.include_configs: + log.info("Downloading centralised configs from GitHub") def prompt_pipeline_name(self): """Prompt for the pipeline name if not set with a flag""" @@ -231,10 +242,13 @@ def prompt_revision(self): # If --tower is specified, allow to select multiple revisions if not bool(self.revision): - self.revision.extend( - nf_core.utils.prompt_pipeline_release_branch(self.wf_revisions, self.wf_branches, multiple=self.tower) + temp = nf_core.utils.prompt_pipeline_release_branch( + self.wf_revisions, self.wf_branches, multiple=self.tower ) + # have to make sure that self.revision is a list of strings, regardless if temp is str or list of strings. + self.revision.append(temp) if isinstance(temp, str) else self.revision.extend(temp) + def get_revision_hash(self): """Find specified revision / branch hash""" @@ -270,6 +284,13 @@ def get_revision_hash(self): # Set the download URL and return - only applicable for classic downloads self.wf_download_url = f"https://github.com/{self.pipeline}/archive/{list(self.wf_sha.values())[0]}.zip" + def prompt_config_inclusion(self): + """Prompt for inclusion of institutional configurations""" + self.include_configs = questionary.confirm( + "Include the nf-core's default institutional configuration files into the download?", + style=nf_core.utils.nfcore_question_style, + ).ask() + def prompt_container_download(self): """Prompt whether to download container images or not""" @@ -855,12 +876,16 @@ def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=Fa Args: remote_url (str): The URL of the remote repository. Defaults to None. - self.revision (list): The revision to use. A list of strings. - commit (str): The commit to clone. Defaults to None. + self.revision (list of str): The revision to use. A list of strings. + commit (dict of str): The commit to clone. Defaults to None. no_pull (bool, optional): Whether to skip the pull step. Defaults to False. hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. """ + import pdb + + pdb.set_trace() + self.remote_url = remote_url self.revision = [].extend(revision) if revision else [] self.commit = [].extend(commit) if commit else [] From 6d04ec8287e2bb4f47aa373beeadb149cb75e0bb Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Sat, 4 Mar 2023 15:23:43 +0100 Subject: [PATCH 066/249] Inclusion of the revision in the output file name is problematic with the new ability to download multiple revisions at once. This resulted in loooooooong filenames. --- nf_core/download.py | 37 ++++++++++++++++++++++++++++--------- nf_core/utils.py | 18 +++++++++++++----- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 41b1fe1458..f6646d29ab 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -3,6 +3,7 @@ from __future__ import print_function import concurrent.futures +from datetime import datetime import git from git.exc import GitCommandError, InvalidGitRepositoryError import io @@ -139,7 +140,10 @@ def download_workflow(self): log.critical(e) sys.exit(1) - summary_log = [f"Pipeline revision: '{','.join(self.revision)}'", f"Pull containers: '{self.container}'"] + summary_log = [ + f"Pipeline revision: '{','.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", + f"Pull containers: '{self.container}'", + ] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: summary_log.append(f"Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {os.environ['NXF_SINGULARITY_CACHEDIR']}") @@ -242,12 +246,29 @@ def prompt_revision(self): # If --tower is specified, allow to select multiple revisions if not bool(self.revision): - temp = nf_core.utils.prompt_pipeline_release_branch( + (choice, tag_set) = nf_core.utils.prompt_pipeline_release_branch( self.wf_revisions, self.wf_branches, multiple=self.tower ) - # have to make sure that self.revision is a list of strings, regardless if temp is str or list of strings. - self.revision.append(temp) if isinstance(temp, str) else self.revision.extend(temp) + # The checkbox() prompt unfortunately does not support passing a Validator, + # so a user who keeps pressing Enter will bump through the selection + + # bool(choice), bool(tag_set): + ############################# + # True, True: A choice was made and revisions were available. + # False, True: No selection was made, but revisions were available -> defaults to all available. + # False, False: No selection was made because no revisions were available -> raise AssertionError. + # True, False: Congratulations, you found a bug! That combo shouldn't happen. + + if bool(choice): + # have to make sure that self.revision is a list of strings, regardless if temp is str or list of strings. + self.revision.append(choice) if isinstance(choice, str) else self.revision.extend(choice) + else: + if bool(tag_set): + self.revision = tag_set + log.info("No particular revision was selected, all available will be downloaded.") + else: + raise AssertionError(f"No revisions of {self.pipeline} available for download.") def get_revision_hash(self): """Find specified revision / branch hash""" @@ -276,9 +297,7 @@ def get_revision_hash(self): # Set the outdir if not self.outdir: - self.outdir = ( - f"{self.pipeline.replace('/', '-').lower()}-{'_'.join(self.revision) if self.revision else ''}" - ) + self.outdir = f"{self.pipeline.replace('/', '-').lower()}_{datetime.now().strftime('%Y-%m-%d_%H-%M')}" if not self.tower and bool(self.wf_sha): # Set the download URL and return - only applicable for classic downloads @@ -876,8 +895,8 @@ def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=Fa Args: remote_url (str): The URL of the remote repository. Defaults to None. - self.revision (list of str): The revision to use. A list of strings. - commit (dict of str): The commit to clone. Defaults to None. + self.revision (list of str): The revisions to include. A list of strings. + commits (dict of str): The checksums to linked with the revisions. no_pull (bool, optional): Whether to skip the pull step. Defaults to False. hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. diff --git a/nf_core/utils.py b/nf_core/utils.py index cef1cf3272..f73d7c9dc4 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -834,29 +834,37 @@ def prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False): Returns: choice (str): Selected release / branch name """ - # Prompt user for release tag + # Prompt user for release tag, tag_set will contain all available. choices = [] + tag_set = [] # Releases if len(wf_releases) > 0: for tag in map(lambda release: release.get("tag_name"), wf_releases): tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] choices.append(questionary.Choice(title=tag_display, value=tag)) + tag_set.append(tag) # Branches for branch in wf_branches.keys(): branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] choices.append(questionary.Choice(title=branch_display, value=branch)) + tag_set.append(branch) if len(choices) == 0: return False if multiple: - return questionary.checkbox( - "Select release / branch:", choices=choices, style=nfcore_question_style - ).unsafe_ask() + return ( + questionary.checkbox("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask(), + tag_set, + ) + else: - return questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask() + return ( + questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask(), + tag_set, + ) def get_repo_releases_branches(pipeline, wfs): From 7642e4fb86384540175febd25b4919e0a21dde0e Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 8 Mar 2023 18:33:17 +0100 Subject: [PATCH 067/249] Allow multiple instances of the -r argument. Needed for scripted download. Ultimately, this now means that I also have to implement multiple version downloads for the classic download. Just downloading the first doesn't seem to make sense from a UX perspective. --- nf_core/__main__.py | 7 ++++++- nf_core/download.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 521454eb99..056242aac2 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -209,7 +209,12 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all # nf-core download @nf_core_cli.command() @click.argument("pipeline", required=False, metavar="") -@click.option("-r", "--revision", type=str, help="Pipeline release") +@click.option( + "-r", + "--revision", + multiple=True, + help="Pipeline release to download. Multiple invocations are possible.", +) @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( "-x", "--compress", type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]), help="Archive compression type" diff --git a/nf_core/download.py b/nf_core/download.py index f6646d29ab..61d4f2c1c8 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -95,7 +95,12 @@ def __init__( parallel_downloads=4, ): self.pipeline = pipeline - self.revision = [].extend(revision) if revision else [] + if isinstance(revision, str): + self.revision = [revision] + elif isinstance(revision, tuple): + self.revision = [*revision] + else: + self.revision = [] self.outdir = outdir self.output_filename = None self.compress_type = compress_type From 7f93edbc81bb267c2a0b9f5b85feb7c8f98abbea Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 28 Mar 2023 16:02:23 +0200 Subject: [PATCH 068/249] Finished updating the prompts for the dialogues. --- nf_core/download.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 61d4f2c1c8..ad8da971d1 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -133,8 +133,8 @@ def download_workflow(self): ) self.prompt_revision() self.get_revision_hash() - # inclusion of configs is unsuitable for multi-revision repositories. - if len(self.revision) == 1: + # inclusion of configs is unnecessary for Tower. + if not self.tower: self.prompt_config_inclusion() self.prompt_container_download() self.prompt_use_singularity_cachedir() @@ -146,7 +146,7 @@ def download_workflow(self): sys.exit(1) summary_log = [ - f"Pipeline revision: '{','.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", + f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", f"Pull containers: '{self.container}'", ] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: @@ -187,12 +187,6 @@ def download_workflow(self): # Summary log log.info("Saving '{}'\n {}".format(self.pipeline, "\n ".join(summary_log))) - # Actually download the workflow - if not self.tower: - self.download_workflow_classic() - else: - self.download_workflow_tower() - def download_workflow_classic(self): """Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner.""" # Download the pipeline files @@ -256,7 +250,7 @@ def prompt_revision(self): ) # The checkbox() prompt unfortunately does not support passing a Validator, - # so a user who keeps pressing Enter will bump through the selection + # so a user who keeps pressing Enter will bump through the selection without choice. # bool(choice), bool(tag_set): ############################# @@ -266,7 +260,7 @@ def prompt_revision(self): # True, False: Congratulations, you found a bug! That combo shouldn't happen. if bool(choice): - # have to make sure that self.revision is a list of strings, regardless if temp is str or list of strings. + # have to make sure that self.revision is a list of strings, regardless if choice is str or list of strings. self.revision.append(choice) if isinstance(choice, str) else self.revision.extend(choice) else: if bool(tag_set): @@ -302,7 +296,10 @@ def get_revision_hash(self): # Set the outdir if not self.outdir: - self.outdir = f"{self.pipeline.replace('/', '-').lower()}_{datetime.now().strftime('%Y-%m-%d_%H-%M')}" + if len(self.wf_sha) > 1: + self.outdir = f"{self.pipeline.replace('/', '-').lower()}_{datetime.now().strftime('%Y-%m-%d_%H-%M')}" + else: + self.outdir = f"{self.pipeline.replace('/', '-').lower()}_{self.revision[0]}" if not self.tower and bool(self.wf_sha): # Set the download URL and return - only applicable for classic downloads From 12bf9428868e1e4af2fd408ba4a2c8d05643988f Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 13 Apr 2023 17:37:20 +0200 Subject: [PATCH 069/249] Converted the self.wf_download_url into a dict. --- nf_core/download.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index ad8da971d1..8e2ead0f86 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -114,7 +114,7 @@ def __init__( self.wf_revisions = {} self.wf_branches = {} self.wf_sha = {} - self.wf_download_url = None + self.wf_download_url = {} self.nf_config = {} self.containers = [] @@ -162,7 +162,7 @@ def download_workflow(self): else: summary_log.append(f"Output directory: '{self.outdir}'") - if len(self.revision) == 1: + if not self.tower: # Only show entry, if option was prompted. summary_log.append(f"Include default institutional configuration: '{self.include_configs}'") @@ -187,8 +187,20 @@ def download_workflow(self): # Summary log log.info("Saving '{}'\n {}".format(self.pipeline, "\n ".join(summary_log))) + # Perform the actual download + if self.tower: + # self.download_workflow_tower() + pass + else: + self.download_workflow_classic() + def download_workflow_classic(self): """Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner.""" + + import pdb + + pdb.set_trace() + # Download the pipeline files log.info("Downloading workflow files from GitHub") self.download_wf_files() @@ -301,9 +313,13 @@ def get_revision_hash(self): else: self.outdir = f"{self.pipeline.replace('/', '-').lower()}_{self.revision[0]}" - if not self.tower and bool(self.wf_sha): - # Set the download URL and return - only applicable for classic downloads - self.wf_download_url = f"https://github.com/{self.pipeline}/archive/{list(self.wf_sha.values())[0]}.zip" + if not self.tower: + for revision, wf_sha in self.wf_sha.items(): + # Set the download URL and return - only applicable for classic downloads + self.wf_download_url = { + **self.wf_download_url, + revision: f"https://github.com/{self.pipeline}/archive/{wf_sha}.zip", + } def prompt_config_inclusion(self): """Prompt for inclusion of institutional configurations""" @@ -903,9 +919,6 @@ def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=Fa hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. """ - import pdb - - pdb.set_trace() self.remote_url = remote_url self.revision = [].extend(revision) if revision else [] From 2ff62f31b4ec9aa7998f04bd13ebf1e25b0855b5 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 14 Apr 2023 17:08:09 +0200 Subject: [PATCH 070/249] Enable multi-revision classic download. --- nf_core/download.py | 135 ++++++++++++++++++++++++++++++-------------- 1 file changed, 93 insertions(+), 42 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 8e2ead0f86..3327f7580d 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -197,28 +197,32 @@ def download_workflow(self): def download_workflow_classic(self): """Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner.""" - import pdb - - pdb.set_trace() - - # Download the pipeline files - log.info("Downloading workflow files from GitHub") - self.download_wf_files() - - # Download the centralised configs + # Download the centralised configs first if self.include_configs: log.info("Downloading centralised configs from GitHub") self.download_configs() - try: - self.wf_use_local_configs() - except FileNotFoundError as e: - log.error("Error editing pipeline config file to use local configs!") - log.critical(e) - sys.exit(1) + + # Download the pipeline files for each selected revision + log.info("Downloading workflow files from GitHub") + + for item in zip(self.revision, self.wf_sha.values(), self.wf_download_url.values()): + revision_dirname = self.download_wf_files(revision=item[0], wf_sha=item[1], download_url=item[2]) + + if self.include_configs: + try: + self.wf_use_local_configs(revision_dirname) + except FileNotFoundError as e: + log.error("Error editing pipeline config file to use local configs!") + log.critical(e) + sys.exit(1) + + # Collect all required singularity images + if self.container == "singularity": + self.find_container_images(revision_dirname) # Download the singularity images if self.container == "singularity": - self.find_container_images() + log.info(f"Found {len(self.containers)} container{'s' if len(self.containers) > 1 else ''}") try: self.get_singularity_images() except OSError as e: @@ -442,24 +446,29 @@ def prompt_compression_type(self): if self.compress_type == "none": self.compress_type = None - def download_wf_files(self): + def download_wf_files(self, revision, wf_sha, download_url): """Downloads workflow files from GitHub to the :attr:`self.outdir`.""" - log.debug(f"Downloading {self.wf_download_url}") + log.debug(f"Downloading {download_url}") # Download GitHub zip file into memory and extract - url = requests.get(self.wf_download_url) + url = requests.get(download_url) with ZipFile(io.BytesIO(url.content)) as zipfile: zipfile.extractall(self.outdir) + # create a filesystem-safe version of the revision name for the directory + revision_dirname = re.sub("[^0-9a-zA-Z]+", "_", revision) + # Rename the internal directory name to be more friendly - gh_name = f"{self.pipeline}-{list(self.wf_sha.values())[0] if bool(self.wf_sha) else ''}".split("/")[-1] - os.rename(os.path.join(self.outdir, gh_name), os.path.join(self.outdir, "workflow")) + gh_name = f"{self.pipeline}-{wf_sha if bool(wf_sha) else ''}".split("/")[-1] + os.rename(os.path.join(self.outdir, gh_name), os.path.join(self.outdir, revision_dirname)) # Make downloaded files executable - for dirpath, _, filelist in os.walk(os.path.join(self.outdir, "workflow")): + for dirpath, _, filelist in os.walk(os.path.join(self.outdir, revision_dirname)): for fname in filelist: os.chmod(os.path.join(dirpath, fname), 0o775) + return revision_dirname + def download_configs(self): """Downloads the centralised config profiles from nf-core/configs to :attr:`self.outdir`.""" configs_zip_url = "https://github.com/nf-core/configs/archive/master.zip" @@ -479,9 +488,9 @@ def download_configs(self): for fname in filelist: os.chmod(os.path.join(dirpath, fname), 0o775) - def wf_use_local_configs(self): + def wf_use_local_configs(self, revision_dirname): """Edit the downloaded nextflow.config file to use the local config files""" - nfconfig_fn = os.path.join(self.outdir, "workflow", "nextflow.config") + nfconfig_fn = os.path.join(self.outdir, revision_dirname, "nextflow.config") find_str = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" repl_str = "${projectDir}/../configs/" log.debug(f"Editing 'params.custom_config_base' in '{nfconfig_fn}'") @@ -507,7 +516,7 @@ def wf_use_local_configs(self): with open(nfconfig_fn, "w") as nfconfig_fh: nfconfig_fh.write(nfconfig) - def find_container_images(self): + def find_container_images(self, revision_dirname): """Find container image names for workflow. Starts by using `nextflow config` to pull out any process.container @@ -533,15 +542,23 @@ def find_container_images(self): 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : 'biocontainers/fastqc:0.11.9--0' }" + Later DSL2, variable is being used: + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + "https://depot.galaxyproject.org/singularity/${container_id}" : + "quay.io/biocontainers/${container_id}" }" + + container_id = 'mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0' + DSL1 / Special case DSL2: container "nfcore/cellranger:6.0.2" """ log.debug("Fetching container names for workflow") - containers_raw = [] + # since this is run for multiple versions now, account for previous invocations + containers_raw = [] if not self.containers else self.containers # Use linting code to parse the pipeline nextflow config - self.nf_config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, "workflow")) + self.nf_config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, revision_dirname)) # Find any config variables that look like a container for k, v in self.nf_config.items(): @@ -549,7 +566,7 @@ def find_container_images(self): containers_raw.append(v.strip('"').strip("'")) # Recursive search through any DSL2 module files for container spec lines. - for subdir, _, files in os.walk(os.path.join(self.outdir, "workflow", "modules")): + for subdir, _, files in os.walk(os.path.join(self.outdir, revision_dirname, "modules")): for file in files: if file.endswith(".nf"): file_path = os.path.join(subdir, file) @@ -569,18 +586,54 @@ def find_container_images(self): break # Prioritise http, exit loop as soon as we find it # No https download, is the entire container string a docker URI? - else: - # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/39672069/713980 - docker_regex = r"^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(? 1 else ''}") - def get_singularity_images(self): """Loop through container names and download Singularity images""" From 986f791a438315eec0ae49190d3abda381a7e8ef Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Mon, 17 Apr 2023 15:47:00 +0200 Subject: [PATCH 071/249] Small tweaks to ensure that tools doesn't bail out if there is no symlink from singularity to apptainer on the system. --- nf_core/download.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 3327f7580d..a09e457e36 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -689,8 +689,11 @@ def get_singularity_images(self): containers_pull.append([container, out_path, cache_path]) # Exit if we need to pull images and Singularity is not installed - if len(containers_pull) > 0 and shutil.which("singularity") is None: - raise OSError("Singularity is needed to pull images, but it is not installed") + if len(containers_pull) > 0: + if not shutil.which("singularity") or not shutil.which("apptainer"): + raise OSError( + "Singularity/Apptainer is needed to pull images, but it is not installed or not in $PATH" + ) # Go through each method of fetching containers in order for container in containers_exist: @@ -881,7 +884,12 @@ def singularity_pull_image(self, container, out_path, cache_path, progress): # Pull using singularity address = f"docker://{container.replace('docker://', '')}" - singularity_command = ["singularity", "pull", "--name", output_path, address] + if shutil.which("singularity"): + singularity_command = ["singularity", "pull", "--name", output_path, address] + elif shutil.which("apptainer"): + singularity_command = ["apptainer", "pull", "--name", output_path, address] + else: + raise OSError("Singularity/Apptainer is needed to pull images, but it is not installed or not in $PATH") log.debug(f"Building singularity image: {address}") log.debug(f"Singularity command: {' '.join(singularity_command)}") From 6f95829a2269be9d43c488a84c4a9c9d06344a02 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 18 Apr 2023 18:33:49 +0200 Subject: [PATCH 072/249] Initialise the Git repo clone of the workflow. --- nf_core/download.py | 80 +++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 50 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index a09e457e36..8c66517972 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -189,8 +189,7 @@ def download_workflow(self): # Perform the actual download if self.tower: - # self.download_workflow_tower() - pass + self.download_workflow_tower() else: self.download_workflow_classic() @@ -242,9 +241,14 @@ def download_workflow_tower(self): self.workflow_repo = WorkflowRepo( remote_url=f"git@github.com:{self.pipeline}.git", revision=self.revision if self.revision else None, - commit=self.wf_sha.values if bool(self.wf_sha) else None, + commit=self.wf_sha.values() if bool(self.wf_sha) else None, + in_cache=False, ) + import pdb + + pdb.set_trace() + if self.include_configs: log.info("Downloading centralised configs from GitHub") @@ -457,6 +461,9 @@ def download_wf_files(self, revision, wf_sha, download_url): # create a filesystem-safe version of the revision name for the directory revision_dirname = re.sub("[^0-9a-zA-Z]+", "_", revision) + # account for name collisions, if there is a branch / release named "configs" or "singularity-images" + if revision_dirname in ["configs", "singularity-images"]: + revision_dirname = re.sub("[^0-9a-zA-Z]+", "_", self.pipeline + revision_dirname) # Rename the internal directory name to be more friendly gh_name = f"{self.pipeline}-{wf_sha if bool(wf_sha) else ''}".split("/")[-1] @@ -980,16 +987,25 @@ def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=Fa """ self.remote_url = remote_url - self.revision = [].extend(revision) if revision else [] - self.commit = [].extend(commit) if commit else [] - self.hide_progress = hide_progress + if isinstance(revision, str): + self.revision = [revision] + elif isinstance(revision, list): + self.revision = [*revision] + else: + self.revision = [] + if isinstance(commit, str): + self.commit = [commit] + elif isinstance(revision, list): + self.commit = [*commit] + else: + self.commit = [] self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) - self.setup_local_repo(remote_url, commit, hide_progress, in_cache=in_cache) + self.setup_local_repo(remote_url, commit=None, in_cache=in_cache) def __repr__(self): """Called by print, creates representation of object""" - return f"" + return f"" def retry_setup_local_repo(self): if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): @@ -999,7 +1015,7 @@ def retry_setup_local_repo(self): else: raise LookupError("Exiting due to error with local modules git repo") - def setup_local_repo(self, remote, commit, hide_progress=False, in_cache=True): + def setup_local_repo(self, remote, commit=None, in_cache=True): """ Sets up the local git repository. If the repository has been cloned previously, it returns a git.Repo object of that clone. Otherwise it tries to clone the repository from @@ -1007,7 +1023,7 @@ def setup_local_repo(self, remote, commit, hide_progress=False, in_cache=True): Args: remote (str): git url of remote - branch (str): name of branch to use + commit (str): name of branch to checkout from (optional) hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. Sets self.repo @@ -1022,7 +1038,7 @@ def setup_local_repo(self, remote, commit, hide_progress=False, in_cache=True): rich.progress.BarColumn(bar_width=None), "[bold yellow]{task.fields[state]}", transient=True, - disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, + disable=os.environ.get("HIDE_PROGRESS", None) is not None, ) with pbar: self.repo = git.Repo.clone_from( @@ -1045,7 +1061,7 @@ def setup_local_repo(self, remote, commit, hide_progress=False, in_cache=True): rich.progress.BarColumn(bar_width=None), "[bold yellow]{task.fields[state]}", transient=True, - disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, + disable=os.environ.get("HIDE_PROGRESS", None) is not None, ) with pbar: self.repo.remotes.origin.fetch( @@ -1057,41 +1073,5 @@ def setup_local_repo(self, remote, commit, hide_progress=False, in_cache=True): log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") self.retry_setup_local_repo() finally: - self.repo.git.checkout(commit) - - def add_nfcore_configs(self, commit, hide_progress=False): - """ - Pulls the configuration profiles from the nf-core/config repository on GitHub. - - Args: - commit: The config version to pull - hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. - Sets self.repo - """ - - try: - if os.path.exists(self.local_repo_dir): - try: - pbar = rich.progress.Progress( - "[bold blue]{task.description}", - rich.progress.BarColumn(bar_width=None), - "[bold yellow]{task.fields[state]}", - transient=True, - disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, - ) - with pbar: - self.configs = git.Submodule.add( - self.repo, - "nf-core configuration", - "./conf_institutional", - f"git@github.com:nf-core/configs.git", - progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Adding configuration"), - ) - except GitCommandError: - raise LookupError(f"Failed to retrieve configuration: `{remote}`") - - except (GitCommandError, InvalidGitRepositoryError) as e: - log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") - self.retry_setup_local_repo() - finally: - self.repo.git.checkout(commit) + if commit: + self.repo.git.checkout(commit) From 760fcaad48032aea7232e99bfdcb205d694f66e6 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 19 Apr 2023 18:25:01 +0200 Subject: [PATCH 073/249] WorkflowRepo attributes and functions. --- nf_core/download.py | 68 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 8c66517972..43a27e2f51 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -245,12 +245,29 @@ def download_workflow_tower(self): in_cache=False, ) - import pdb + # Remove tags for those revisions that had not been selected + self.workflow_repo.tidy_tags() - pdb.set_trace() + # extract the required containers + if self.container == "singularity": + for commit in self.wf_sha.values(): + # Checkout the repo in the current revision + self.workflow_repo.checkout(commit) + # Collect all required singularity images + self.find_container_images(self.workflow_repo.access()) + + # Download the singularity images + log.info(f"Found {len(self.containers)} container{'s' if len(self.containers) > 1 else ''}") + try: + self.get_singularity_images() + except OSError as e: + log.critical(f"[red]{e}[/]") + sys.exit(1) - if self.include_configs: - log.info("Downloading centralised configs from GitHub") + # Compress into an archive + if self.compress_type is not None: + log.info("Compressing images") + self.compress_download() def prompt_pipeline_name(self): """Prompt for the pipeline name if not set with a flag""" @@ -973,7 +990,15 @@ class WorkflowRepo(SyncedRepo): """ - def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=False, in_cache=True): + def __init__( + self, + remote_url, + revision, + commit, + no_pull=False, + hide_progress=False, + in_cache=True, + ): """ Initializes the object and clones the workflows git repository if it is not already present @@ -985,7 +1010,6 @@ def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=Fa hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. """ - self.remote_url = remote_url if isinstance(revision, str): self.revision = [revision] @@ -1001,11 +1025,23 @@ def __init__(self, remote_url, revision, commit, no_pull=False, hide_progress=Fa self.commit = [] self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) - self.setup_local_repo(remote_url, commit=None, in_cache=in_cache) + self.setup_local_repo(remote_url, in_cache=in_cache) + + # expose some instance attributes + self.tags = self.repo.tags def __repr__(self): """Called by print, creates representation of object""" - return f"" + return f"" + + def access(self): + if os.path.exists(self.local_repo_dir): + return self.local_repo_dir + else: + return None + + def checkout(self, commit): + return super().checkout(commit) def retry_setup_local_repo(self): if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): @@ -1015,7 +1051,7 @@ def retry_setup_local_repo(self): else: raise LookupError("Exiting due to error with local modules git repo") - def setup_local_repo(self, remote, commit=None, in_cache=True): + def setup_local_repo(self, remote, in_cache=True): """ Sets up the local git repository. If the repository has been cloned previously, it returns a git.Repo object of that clone. Otherwise it tries to clone the repository from @@ -1072,6 +1108,14 @@ def setup_local_repo(self, remote, commit=None, in_cache=True): except (GitCommandError, InvalidGitRepositoryError) as e: log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") self.retry_setup_local_repo() - finally: - if commit: - self.repo.git.checkout(commit) + + def tidy_tags(self): + """ + Function to delete all tags that point to revisions that are not of interest to the downloader. + This allows a clutter-free experience in Tower. The commits are evidently still available. + """ + if self.revision and self.repo and self.repo.tags: + for tag in self.repo.tags: + if tag.name not in self.revision: + self.repo.delete_tag(tag) + self.tags = self.repo.tags From c381776fe37b4824789ba9e3d6fc31a1326ec8c6 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 21 Apr 2023 16:06:57 +0200 Subject: [PATCH 074/249] Finished the Tower download branch. --- nf_core/download.py | 76 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 16 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 43a27e2f51..e8fafa5b1a 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -133,12 +133,13 @@ def download_workflow(self): ) self.prompt_revision() self.get_revision_hash() - # inclusion of configs is unnecessary for Tower. + # Inclusion of configs is unnecessary for Tower. if not self.tower: self.prompt_config_inclusion() self.prompt_container_download() self.prompt_use_singularity_cachedir() self.prompt_singularity_cachedir_only() + # Nothing meaningful to compress here. if not self.tower: self.prompt_compression_type() except AssertionError as e: @@ -230,7 +231,7 @@ def download_workflow_classic(self): # Compress into an archive if self.compress_type is not None: - log.info("Compressing download..") + log.info("Compressing output into archive") self.compress_download() def download_workflow_tower(self): @@ -248,6 +249,9 @@ def download_workflow_tower(self): # Remove tags for those revisions that had not been selected self.workflow_repo.tidy_tags() + # create a bare clone of the modified repository needed for Tower + self.workflow_repo.bare_clone(os.path.join(self.outdir, self.output_filename)) + # extract the required containers if self.container == "singularity": for commit in self.wf_sha.values(): @@ -264,10 +268,9 @@ def download_workflow_tower(self): log.critical(f"[red]{e}[/]") sys.exit(1) - # Compress into an archive - if self.compress_type is not None: - log.info("Compressing images") - self.compress_download() + # Justify why compression is skipped for Tower downloads (Prompt is not shown, but CLI argument could have been set) + if self.compress_type is not None: + log.info("Compression choice is ignored for Tower downloads since nothing can be reasonably compressed.") def prompt_pipeline_name(self): """Prompt for the pipeline name if not set with a flag""" @@ -1019,11 +1022,12 @@ def __init__( self.revision = [] if isinstance(commit, str): self.commit = [commit] - elif isinstance(revision, list): + elif isinstance(commit, list): self.commit = [*commit] else: self.commit = [] self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) + self.retries = 0 # retries for setting up the locally cached repository self.setup_local_repo(remote_url, in_cache=in_cache) @@ -1043,13 +1047,24 @@ def access(self): def checkout(self, commit): return super().checkout(commit) - def retry_setup_local_repo(self): - if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): - log.info(f"Removing '{self.local_repo_dir}'") + def retry_setup_local_repo(self, skip_confirm=False): + self.retries += 1 + if skip_confirm or rich.prompt.Confirm.ask( + f"[violet]Delete local cache '{self.local_repo_dir}' and try again?" + ): + if ( + self.retries > 1 + ): # One unconfirmed retry is acceptable, but prevent infinite loops without user interaction. + log.error( + f"Errors with locally cached repository of '{self.fullname}'. Please delete '{self.local_repo_dir}' manually and try again." + ) + sys.exit(1) + if not skip_confirm: # Feedback to user for manual confirmation. + log.info(f"Removing '{self.local_repo_dir}'") shutil.rmtree(self.local_repo_dir) - self.setup_local_repo(self.remote, self.commit, self.hide_progress) + self.setup_local_repo(self.remote_url, in_cache=False) else: - raise LookupError("Exiting due to error with local modules git repo") + raise LookupError("Exiting due to error with locally cached Git repository.") def setup_local_repo(self, remote, in_cache=True): """ @@ -1113,9 +1128,38 @@ def tidy_tags(self): """ Function to delete all tags that point to revisions that are not of interest to the downloader. This allows a clutter-free experience in Tower. The commits are evidently still available. + + However, due to local caching, the downloader might also want access to revisions that had been deleted before. + In that case, don't bother with re-adding the tags and rather download anew from Github. """ if self.revision and self.repo and self.repo.tags: - for tag in self.repo.tags: - if tag.name not in self.revision: - self.repo.delete_tag(tag) - self.tags = self.repo.tags + desired_tags = self.revision.copy() + try: + for tag in self.repo.tags: + if tag.name not in self.revision: + self.repo.delete_tag(tag) + else: + desired_tags.remove(tag.name) + self.tags = self.repo.tags + if len(desired_tags) > 0: + log.info( + f"Locally cached version of the pipeline lacks selected revisions {', '.join(desired_tags)}. Downloading anew from GitHub..." + ) + self.retry_setup_local_repo(skip_confirm=True) + self.tidy_tags() + except (GitCommandError, InvalidGitRepositoryError) as e: + log.error(f"[red]Adapting your pipeline download unfortunately failed:[/]\n{e}\n") + self.retry_setup_local_repo(skip_confirm=True) + sys.exit(1) + + def bare_clone(self, destination): + if self.repo: + try: + destfolder = os.path.abspath(destination) + if not os.path.exists(destfolder): + os.makedirs(destfolder) + if os.path.exists(destination): + shutil.rmtree(os.path.abspath(destination)) + self.repo.clone(os.path.abspath(destination), bare=True) + except (OSError, GitCommandError, InvalidGitRepositoryError) as e: + log.error(f"[red]Failure to create the pipeline download[/]\n{e}\n") From 526a26e45ddc00fb4ef7969c5df822def07cb0c4 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Mon, 24 Apr 2023 14:55:24 +0200 Subject: [PATCH 075/249] Minor tweaks to the container download functionality. --- nf_core/download.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index e8fafa5b1a..cba7eb7dfb 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -235,7 +235,7 @@ def download_workflow_classic(self): self.compress_download() def download_workflow_tower(self): - """Create a bare-cloned git repository of the workflow that includes the configurations, such it can be launched with `tw launch` as file:/ pipeline""" + """Create a bare-cloned git repository of the workflow, such it can be launched with `tw launch` as file:/ pipeline""" log.info("Collecting workflow from GitHub") @@ -261,12 +261,12 @@ def download_workflow_tower(self): self.find_container_images(self.workflow_repo.access()) # Download the singularity images - log.info(f"Found {len(self.containers)} container{'s' if len(self.containers) > 1 else ''}") - try: - self.get_singularity_images() - except OSError as e: - log.critical(f"[red]{e}[/]") - sys.exit(1) + log.info(f"Found {len(self.containers)} container{'s' if len(self.containers) > 1 else ''}") + try: + self.get_singularity_images() + except OSError as e: + log.critical(f"[red]{e}[/]") + sys.exit(1) # Justify why compression is skipped for Tower downloads (Prompt is not shown, but CLI argument could have been set) if self.compress_type is not None: @@ -581,7 +581,7 @@ def find_container_images(self, revision_dirname): """ log.debug("Fetching container names for workflow") - # since this is run for multiple versions now, account for previous invocations + # since this is run for multiple revisions now, account for previously detected containers. containers_raw = [] if not self.containers else self.containers # Use linting code to parse the pipeline nextflow config From f4b9e673b3b5b3a14175845d4b6d2d71ace759a2 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Mon, 24 Apr 2023 16:19:27 +0200 Subject: [PATCH 076/249] Updating docs and changelog, fixing linting errors. --- CHANGELOG.md | 4 ++++ README.md | 7 +++++++ nf_core/download.py | 8 ++++---- nf_core/modules/modules_repo.py | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b79fd410dc..d880e0105d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,10 @@ - Removed `quay.io` from all module Docker container references as this is now supplied at pipeline level. ([#2249](https://github.com/nf-core/tools/pull/2249)) - Remove `CITATION.cff` file from pipeline template, to avoid that pipeline Zenodo entries reference the nf-core publication instead of the pipeline ([#2059](https://github.com/nf-core/tools/pull/2059)). +### Download + +- Introduce a `--tower` flag for `nf-core download` to obtain pipelines in an offline format suited for [seqeralabs® Nextflow Tower](https://cloud.tower.nf/) ([#2247](https://github.com/nf-core/tools/pull/2247)). + ### Linting - Update modules lint test to fail if enable_conda is found ([#2213](https://github.com/nf-core/tools/pull/2213)) diff --git a/README.md b/README.md index 0de42e86e8..13d8b381a3 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ A python package with helper tools for the nf-core community. - [`nf-core list` - List available pipelines](#listing-pipelines) - [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) - [`nf-core download` - Download pipeline for offline use](#downloading-pipelines-for-offline-use) +- [`nf-core download --tower` - Download pipeline for Tower](#downloading-pipelines-for-tower) - [`nf-core licences` - List software licences in a pipeline](#pipeline-software-licences) - [`nf-core create` - Create a new pipeline with the nf-core template](#creating-a-new-pipeline) - [`nf-core lint` - Check pipeline code against nf-core guidelines](#linting-a-workflow) @@ -401,6 +402,12 @@ Note that compressing many GBs of binary files can be slow, so specifying `--com If the download speeds are much slower than your internet connection is capable of, you can set `--parallel-downloads` to a large number to download loads of images at once. +### Adapting downloads to Nextflow Tower + +[seqeralabs® Nextflow Tower](https://cloud.tower.nf/) provides a graphical user interface to oversee pipeline runs, gather statistics and configure compute resources. While pipelines added to _Tower_ are preferably hosted at a Git service, providing them as disconnected, self-reliant repositories is also possible for premises with restricted network access. Choosing the `--tower` flag will download the pipeline in an appropriate form. + +Subsequently, the `*.git` folder can be moved to it's final destination and linked with a pipeline in _Tower_ using the `file:/` prefix. + ## Pipeline software licences Sometimes it's useful to see the software licences of the tools used in a pipeline. diff --git a/nf_core/download.py b/nf_core/download.py index cba7eb7dfb..9fe9c29c9f 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -3,9 +3,6 @@ from __future__ import print_function import concurrent.futures -from datetime import datetime -import git -from git.exc import GitCommandError, InvalidGitRepositoryError import io import logging import os @@ -15,19 +12,22 @@ import sys import tarfile import textwrap +from datetime import datetime from zipfile import ZipFile +import git import questionary import requests import requests_cache import rich import rich.progress +from git.exc import GitCommandError, InvalidGitRepositoryError import nf_core import nf_core.list import nf_core.utils -from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR from nf_core.synced_repo import RemoteProgressbar, SyncedRepo +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR log = logging.getLogger(__name__) stderr = rich.console.Console( diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 20d581af84..152ed7b0c0 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -11,8 +11,8 @@ import nf_core.modules.modules_json import nf_core.modules.modules_utils -from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config from nf_core.synced_repo import RemoteProgressbar, SyncedRepo +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config log = logging.getLogger(__name__) From 2bf14bd6377fa235bf2066acdaa17b10947d2885 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Mon, 24 Apr 2023 21:44:04 +0200 Subject: [PATCH 077/249] Hopefully fixed the existing tests. New ones still need to be written. --- nf_core/download.py | 2 +- tests/test_cli.py | 4 +++- tests/test_download.py | 38 +++++++++++++++++++++----------------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 9fe9c29c9f..53bb744184 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -717,7 +717,7 @@ def get_singularity_images(self): # Exit if we need to pull images and Singularity is not installed if len(containers_pull) > 0: - if not shutil.which("singularity") or not shutil.which("apptainer"): + if not (shutil.which("singularity") or shutil.which("apptainer")): raise OSError( "Singularity/Apptainer is needed to pull images, but it is not installed or not in $PATH" ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0a6b37144d..6f51fe1025 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -165,6 +165,7 @@ def test_cli_download(self, mock_dl): "outdir": "/path/outdir", "compress": "tar.gz", "force": None, + "tower": None, "container": "singularity", "singularity-cache-only": None, "parallel-downloads": 2, @@ -177,10 +178,11 @@ def test_cli_download(self, mock_dl): mock_dl.assert_called_once_with( cmd[-1], - params["revision"], + (params["revision"],), params["outdir"], params["compress"], "force" in params, + "tower" in params, params["container"], "singularity-cache-only" in params, params["parallel-downloads"], diff --git a/tests/test_download.py b/tests/test_download.py index e2ae882394..d1a770a630 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -32,10 +32,10 @@ def test_get_release_hash_release(self): download_obj.wf_branches, ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) download_obj.get_revision_hash() - assert download_obj.wf_sha == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - assert download_obj.outdir == "nf-core-methylseq-1.6" + assert download_obj.wf_sha[download_obj.revision[0]] == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + assert download_obj.outdir == "nf-core-methylseq_1.6" assert ( - download_obj.wf_download_url + download_obj.wf_download_url[download_obj.revision[0]] == "https://github.com/nf-core/methylseq/archive/b3e5e3b95aaf01d98391a62a10a3990c0a4de395.zip" ) @@ -51,10 +51,10 @@ def test_get_release_hash_branch(self): download_obj.wf_branches, ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) download_obj.get_revision_hash() - assert download_obj.wf_sha == "819cbac792b76cf66c840b567ed0ee9a2f620db7" - assert download_obj.outdir == "nf-core-exoseq-dev" + assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.outdir == "nf-core-exoseq_dev" assert ( - download_obj.wf_download_url + download_obj.wf_download_url[download_obj.revision[0]] == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" ) @@ -78,12 +78,16 @@ def test_get_release_hash_non_existent_release(self): def test_download_wf_files(self, outdir): download_obj = DownloadWorkflow(pipeline="nf-core/methylseq", revision="1.6") download_obj.outdir = outdir - download_obj.wf_sha = "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - download_obj.wf_download_url = ( - "https://github.com/nf-core/methylseq/archive/b3e5e3b95aaf01d98391a62a10a3990c0a4de395.zip" + download_obj.wf_sha = {"1.6": "b3e5e3b95aaf01d98391a62a10a3990c0a4de395"} + download_obj.wf_download_url = { + "1.6": "https://github.com/nf-core/methylseq/archive/b3e5e3b95aaf01d98391a62a10a3990c0a4de395.zip" + } + rev = download_obj.download_wf_files( + download_obj.revision[0], + download_obj.wf_sha[download_obj.revision[0]], + download_obj.wf_download_url[download_obj.revision[0]], ) - download_obj.download_wf_files() - assert os.path.exists(os.path.join(outdir, "workflow", "main.nf")) + assert os.path.exists(os.path.join(outdir, rev, "main.nf")) # # Tests for 'download_configs' @@ -118,7 +122,7 @@ def test_wf_use_local_configs(self, tmp_path): download_obj.download_configs() # Test the function - download_obj.wf_use_local_configs() + download_obj.wf_use_local_configs("workflow") wf_config = nf_core.utils.fetch_wf_config(os.path.join(test_outdir, "workflow"), cache_config=False) assert wf_config["params.custom_config_base"] == f"'{test_outdir}/workflow/../configs/'" @@ -133,14 +137,14 @@ def test_find_container_images(self, tmp_path, mock_fetch_wf_config): "process.mapping.container": "cutting-edge-container", "process.nocontainer": "not-so-cutting-edge", } - download_obj.find_container_images() + download_obj.find_container_images("workflow") assert len(download_obj.containers) == 1 assert download_obj.containers[0] == "cutting-edge-container" # # Tests for 'singularity_pull_image' # - # If Singularity is installed, but the container can't be accessed because it does not exist or there are aceess + # If Singularity is installed, but the container can't be accessed because it does not exist or there are access # restrictions, a FileNotFoundError is raised due to the unavailability of the image. @pytest.mark.skipif( shutil.which("singularity") is None, @@ -153,16 +157,16 @@ def test_singularity_pull_image_singularity_installed(self, tmp_dir, mock_rich_p with pytest.raises(FileNotFoundError): download_obj.singularity_pull_image("a-container", tmp_dir, None, mock_rich_progress) - # If Singularity is not installed, it raises a FileNotFoundError because the singularity command can't be found. + # If Singularity is not installed, it raises a OSError because the singularity command can't be found. @pytest.mark.skipif( shutil.which("singularity") is not None, - reason="Can't test how the code behaves when sungularity is not installed if it is.", + reason="Can't test how the code behaves when singularity is not installed if it is.", ) @with_temporary_folder @mock.patch("rich.progress.Progress.add_task") def test_singularity_pull_image_singularity_not_installed(self, tmp_dir, mock_rich_progress): download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_dir) - with pytest.raises(FileNotFoundError): + with pytest.raises(OSError): download_obj.singularity_pull_image("a-container", tmp_dir, None, mock_rich_progress) # From 8de588ecf72df8964edceba689505c07da7a353b Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 25 Apr 2023 14:18:58 +0200 Subject: [PATCH 078/249] Refactor the CLI commands for the Singularity Cache Dir --- nf_core/__main__.py | 36 +++++- nf_core/download.py | 287 +++++++++++++++++++++++++++++--------------- 2 files changed, 220 insertions(+), 103 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 056242aac2..46e9ac0988 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -225,11 +225,30 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all "-c", "--container", type=click.Choice(["none", "singularity"]), help="Download software container images" ) @click.option( - "--singularity-cache-only/--singularity-cache-copy", - help="Don't / do copy images to the output directory and set 'singularity.cacheDir' in workflow", + "-s", + "--singularity-cache", + type=click.Choice(["amend", "copy", "remote"]), + help="Utilize the 'singularity.cacheDir' in the download process, if applicable.", +) +@click.option( + "-i", + "--singularity-cache-index", + type=str, + help="List of images already available in a remote 'singularity.cacheDir', imposes --singularity-cache=remote", ) @click.option("-p", "--parallel-downloads", type=int, default=4, help="Number of parallel image downloads") -def download(pipeline, revision, outdir, compress, force, tower, container, singularity_cache_only, parallel_downloads): +def download( + pipeline, + revision, + outdir, + compress, + force, + tower, + container, + singularity_cache, + singularity_cache_index, + parallel_downloads, +): """ Download a pipeline, nf-core/configs and pipeline singularity images. @@ -239,7 +258,16 @@ def download(pipeline, revision, outdir, compress, force, tower, container, sing from nf_core.download import DownloadWorkflow dl = DownloadWorkflow( - pipeline, revision, outdir, compress, force, tower, container, singularity_cache_only, parallel_downloads + pipeline, + revision, + outdir, + compress, + force, + tower, + container, + singularity_cache, + singularity_cache_index, + parallel_downloads, ) dl.download_workflow() diff --git a/nf_core/download.py b/nf_core/download.py index 53bb744184..59e7f47d3f 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -77,7 +77,7 @@ class DownloadWorkflow: Args: pipeline (str): A nf-core pipeline name. revision (List[str]): The workflow revision to download, like `1.0`. Defaults to None. - singularity (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False. + container (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False. tower (bool): Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. outdir (str): Path to the local download directory. Defaults to None. """ @@ -91,7 +91,8 @@ def __init__( force=False, tower=False, container=None, - singularity_cache_only=False, + singularity_cache=None, + singularity_cache_index=None, parallel_downloads=4, ): self.pipeline = pipeline @@ -106,9 +107,12 @@ def __init__( self.compress_type = compress_type self.force = force self.tower = tower - self.include_configs = True + self.include_configs = None self.container = container - self.singularity_cache_only = singularity_cache_only + self.singularity_cache = ( + singularity_cache if not singularity_cache_index else "remote" + ) # if a singularity_cache_index is given, use the file and overrule choice. + self.singularity_cache_index = singularity_cache_index self.parallel_downloads = parallel_downloads self.wf_revisions = {} @@ -117,6 +121,7 @@ def __init__( self.wf_download_url = {} self.nf_config = {} self.containers = [] + self.containers_remote = [] # stores the remote images provided in the file. # Fetch remote workflows self.wfs = nf_core.list.Workflows() @@ -134,11 +139,16 @@ def download_workflow(self): self.prompt_revision() self.get_revision_hash() # Inclusion of configs is unnecessary for Tower. - if not self.tower: + if not self.tower and self.include_configs is None: self.prompt_config_inclusion() - self.prompt_container_download() - self.prompt_use_singularity_cachedir() - self.prompt_singularity_cachedir_only() + if not self.singularity_cache == "remote": + self.prompt_container_download() + self.prompt_singularity_cachedir_creation() + else: + self.container = "singularity" + self.prompt_singularity_cachedir_utilization() + self.prompt_singularity_cachedir_remote(retry=False) + self.read_remote_containers() # Nothing meaningful to compress here. if not self.tower: self.prompt_compression_type() @@ -220,9 +230,6 @@ def download_workflow_classic(self): if self.container == "singularity": self.find_container_images(revision_dirname) - # Download the singularity images - if self.container == "singularity": - log.info(f"Found {len(self.containers)} container{'s' if len(self.containers) > 1 else ''}") try: self.get_singularity_images() except OSError as e: @@ -260,8 +267,6 @@ def download_workflow_tower(self): # Collect all required singularity images self.find_container_images(self.workflow_repo.access()) - # Download the singularity images - log.info(f"Found {len(self.containers)} container{'s' if len(self.containers) > 1 else ''}") try: self.get_singularity_images() except OSError as e: @@ -280,24 +285,27 @@ def prompt_pipeline_name(self): self.pipeline = nf_core.utils.prompt_remote_pipeline_name(self.wfs) def prompt_revision(self): - """Prompt for pipeline revision / branch""" - # Prompt user for revision tag if '--revision' was not set - # If --tower is specified, allow to select multiple revisions - + """ + Prompt for pipeline revision / branch + Prompt user for revision tag if '--revision' was not set + If --tower is specified, allow to select multiple revisions + Also the classic download allows for multiple revisions, but + """ if not bool(self.revision): (choice, tag_set) = nf_core.utils.prompt_pipeline_release_branch( self.wf_revisions, self.wf_branches, multiple=self.tower ) + """ + The checkbox() prompt unfortunately does not support passing a Validator, + so a user who keeps pressing Enter will flounder past the selection without choice. - # The checkbox() prompt unfortunately does not support passing a Validator, - # so a user who keeps pressing Enter will bump through the selection without choice. - - # bool(choice), bool(tag_set): + bool(choice), bool(tag_set): ############################# - # True, True: A choice was made and revisions were available. - # False, True: No selection was made, but revisions were available -> defaults to all available. - # False, False: No selection was made because no revisions were available -> raise AssertionError. - # True, False: Congratulations, you found a bug! That combo shouldn't happen. + True, True: A choice was made and revisions were available. + False, True: No selection was made, but revisions were available -> defaults to all available. + False, False: No selection was made because no revisions were available -> raise AssertionError. + True, False: Congratulations, you found a bug! That combo shouldn't happen. + """ if bool(choice): # have to make sure that self.revision is a list of strings, regardless if choice is str or list of strings. @@ -351,10 +359,14 @@ def get_revision_hash(self): def prompt_config_inclusion(self): """Prompt for inclusion of institutional configurations""" - self.include_configs = questionary.confirm( - "Include the nf-core's default institutional configuration files into the download?", - style=nf_core.utils.nfcore_question_style, - ).ask() + if stderr.is_interactive: # Use rich auto-detection of interactive shells + self.include_configs = questionary.confirm( + "Include the nf-core's default institutional configuration files into the download?", + style=nf_core.utils.nfcore_question_style, + ).ask() + else: + self.include_configs = False + # do not include by default. def prompt_container_download(self): """Prompt whether to download container images or not""" @@ -367,7 +379,7 @@ def prompt_container_download(self): style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - def prompt_use_singularity_cachedir(self): + def prompt_singularity_cachedir_creation(self): """Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set""" if ( self.container == "singularity" @@ -381,6 +393,7 @@ def prompt_use_singularity_cachedir(self): if rich.prompt.Confirm.ask( "[blue bold]?[/] [bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" ): + self.singularity_cache == "amend" # Prompt user for a cache directory path cachedir_path = None while cachedir_path is None: @@ -425,25 +438,89 @@ def prompt_use_singularity_cachedir(self): "You will need reload your terminal after the download completes for this to take effect." ) - def prompt_singularity_cachedir_only(self): + def prompt_singularity_cachedir_utilization(self): """Ask if we should *only* use $NXF_SINGULARITY_CACHEDIR without copying into target""" if ( - self.singularity_cache_only is None + self.singularity_cache is None # no choice regarding singularity cache has been made. and self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None ): stderr.print( - "\nIf you are working on the same system where you will run Nextflow, you can leave the downloaded images in the " - "[blue not bold]$NXF_SINGULARITY_CACHEDIR[/] folder, Nextflow will automatically find them. " + "\nIf you are working on the same system where you will run Nextflow, you can amend the downloaded images to the ones in the" + "[blue not bold]$NXF_SINGULARITY_CACHEDIR[/] folder, Nextflow will automatically find them." "However if you will transfer the downloaded files to a different system then they should be copied to the target folder." ) - self.singularity_cache_only = rich.prompt.Confirm.ask( - "[blue bold]?[/] [bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder?[/]" + self.singularity_cache = rich.prompt.Prompt.ask( + "[blue bold]?[/] [bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder or amend new images to the collection?[/]", + choices=["amend", "copy"], ) - # Sanity check, for when passed as a cli flag - if self.singularity_cache_only and self.container != "singularity": - raise AssertionError("Command has '--singularity-cache-only' set, but '--container' is not 'singularity'") + def prompt_singularity_cachedir_remote(self, retry): + """Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR""" + if ( + self.container == "singularity" + and self.singularity_cache == "remote" + and self.singularity_cache_index is None + and stderr.is_interactive # Use rich auto-detection of interactive shells + ): + stderr.print( + "\nNextflow and nf-core can use an environment variable called [blue]$NXF_SINGULARITY_CACHEDIR[/] that is a path to a directory where remote Singularity images are stored. " + "This allows downloaded images to be cached in a central location." + ) + # Prompt user for a file listing the contents of the remote cache directory + cachedir_index = None + while cachedir_index is None: + prompt_cachedir_index = questionary.path( + "Specify a list of the remote images already present in the remote system :", + file_filter="*.txt", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + cachedir_index = os.path.abspath(os.path.expanduser(prompt_cachedir_index)) + if prompt_cachedir_index == "": + log.error("Will disregard contents of a remote [blue]$NXF_SINGULARITY_CACHEDIR[/]") + self.singularity_cache_index = None + self.singularity_cache = "copy" + elif not os.access(cachedir_index, os.R_OK): + log.error(f"'{cachedir_index}' is not a valid, readable file.") + cachedir_index = None + if cachedir_index: + self.singularity_cache_index = cachedir_index + if retry: # invoke parsing the file again. + self.read_remote_containers() + + def read_remote_containers(self): + """Reads the file specified as index for the remote Singularity cache dir""" + if ( + self.container == "singularity" + and self.singularity_cache == "remote" + and self.singularity_cache_index is not None + ): + n_total_images = 0 + try: + with open(self.singularity_cache_index) as indexfile: + for line in indexfile.readlines(): + match = re.search(r"([^\/\\]+\.img)", line, re.S) + if match: + n_total_images += 1 + self.containers_remote.append(match.group(0)) + if n_total_images == 0: + raise LookupError("Could not find valid container names in the index file.") + else: + log.info( + f"Successfully read {n_total_images} containers from the remote $NXF_SINGULARITY_CACHE contents." + ) + self.containers_remote = sorted(list(set(self.containers_remote))) + except (FileNotFoundError, LookupError) as e: + log.error(f"[red]Issue with reading the specified remote $NXF_SINGULARITY_CACHE index:[/]\n{e}\n") + if rich.prompt.Confirm.ask(f"[blue]Specify a new index file and try again?"): + self.prompt_singularity_cachedir_remote(retry=True) + else: + log.info("Proceeding without consideration of the remote $NXF_SINGULARITY_CACHE index.") + self.singularity_cache_index = None + if os.environ.get("NXF_SINGULARITY_CACHEDIR"): + self.singularity_cache = "copy" # default to copy if possible, otherwise skip. + else: + self.singularity_cache = None def prompt_compression_type(self): """Ask user if we should compress the downloaded files""" @@ -531,7 +608,7 @@ def wf_use_local_configs(self, revision_dirname): nfconfig = nfconfig.replace(find_str, repl_str) # Append the singularity.cacheDir to the end if we need it - if self.container == "singularity" and not self.singularity_cache_only: + if self.container == "singularity" and self.singularity_cache == "copy": nfconfig += ( f"\n\n// Added by `nf-core download` v{nf_core.__version__} //\n" + 'singularity.cacheDir = "${projectDir}/../singularity-images/"' @@ -674,8 +751,14 @@ def get_singularity_images(self): if len(self.containers) == 0: log.info("No container names found in workflow") else: + log.info( + f"Found {len(self.containers)} container image{'s' if len(self.containers) > 1 else ''} in workflow." + ) + with DownloadProgress() as progress: - task = progress.add_task("all_containers", total=len(self.containers), progress_type="summary") + task = progress.add_task( + "Collecting container images", total=len(self.containers), progress_type="summary" + ) # Organise containers based on what we need to do with them containers_exist = [] @@ -697,8 +780,8 @@ def get_singularity_images(self): log.debug(f"Cache directory not found, creating: {cache_path_dir}") os.makedirs(cache_path_dir) - # We already have the target file in place, return - if os.path.exists(out_path): + # We already have the target file in place or in remote cache, return + if os.path.exists(out_path) or os.path.basename(out_path) in self.containers_remote: containers_exist.append(container) continue @@ -722,56 +805,62 @@ def get_singularity_images(self): "Singularity/Apptainer is needed to pull images, but it is not installed or not in $PATH" ) - # Go through each method of fetching containers in order - for container in containers_exist: - progress.update(task, description="Image file exists") - progress.update(task, advance=1) - - for container in containers_cache: - progress.update(task, description="Copying singularity images from cache") - self.singularity_copy_cache_image(*container) - progress.update(task, advance=1) - - with concurrent.futures.ThreadPoolExecutor(max_workers=self.parallel_downloads) as pool: - progress.update(task, description="Downloading singularity images") - - # Kick off concurrent downloads - future_downloads = [ - pool.submit(self.singularity_download_image, *container, progress) - for container in containers_download - ] - - # Make ctrl-c work with multi-threading - self.kill_with_fire = False - - try: - # Iterate over each threaded download, waiting for them to finish - for future in concurrent.futures.as_completed(future_downloads): - future.result() - try: - progress.update(task, advance=1) - except Exception as e: - log.error(f"Error updating progress bar: {e}") - - except KeyboardInterrupt: - # Cancel the future threads that haven't started yet - for future in future_downloads: - future.cancel() - # Set the variable that the threaded function looks for - # Will trigger an exception from each thread - self.kill_with_fire = True - # Re-raise exception on the main thread - raise - - for container in containers_pull: - progress.update(task, description="Pulling singularity images") - try: - self.singularity_pull_image(*container, progress) - except RuntimeWarning as r: - # Raise exception if this is not possible - log.error("Not able to pull image. Service might be down or internet connection is dead.") - raise r - progress.update(task, advance=1) + if containers_exist: + if self.singularity_cache_index is not None: + log.info(f"{len(containers_exist)} are already cached remotely and won't be retrieved.") + # Go through each method of fetching containers in order + for container in containers_exist: + progress.update(task, description="Image file exists at destination") + progress.update(task, advance=1) + + if containers_cache: + for container in containers_cache: + progress.update(task, description="Copying singularity images from cache") + self.singularity_copy_cache_image(*container) + progress.update(task, advance=1) + + if containers_download or containers_pull: + # if clause gives slightly better UX, because Download is no longer displayed if nothing is left to be downloaded. + with concurrent.futures.ThreadPoolExecutor(max_workers=self.parallel_downloads) as pool: + progress.update(task, description="Downloading singularity images") + + # Kick off concurrent downloads + future_downloads = [ + pool.submit(self.singularity_download_image, *container, progress) + for container in containers_download + ] + + # Make ctrl-c work with multi-threading + self.kill_with_fire = False + + try: + # Iterate over each threaded download, waiting for them to finish + for future in concurrent.futures.as_completed(future_downloads): + future.result() + try: + progress.update(task, advance=1) + except Exception as e: + log.error(f"Error updating progress bar: {e}") + + except KeyboardInterrupt: + # Cancel the future threads that haven't started yet + for future in future_downloads: + future.cancel() + # Set the variable that the threaded function looks for + # Will trigger an exception from each thread + self.kill_with_fire = True + # Re-raise exception on the main thread + raise + + for container in containers_pull: + progress.update(task, description="Pulling singularity images") + try: + self.singularity_pull_image(*container, progress) + except RuntimeWarning as r: + # Raise exception if this is not possible + log.error("Not able to pull image. Service might be down or internet connection is dead.") + raise r + progress.update(task, advance=1) def singularity_image_filenames(self, container): """Check Singularity cache for image, copy to destination folder if found. @@ -810,11 +899,11 @@ def singularity_image_filenames(self, container): if os.environ.get("NXF_SINGULARITY_CACHEDIR"): cache_path = os.path.join(os.environ["NXF_SINGULARITY_CACHEDIR"], out_name) # Use only the cache - set this as the main output path - if self.singularity_cache_only: + if self.singularity_cache == "amend": out_path = cache_path cache_path = None - elif self.singularity_cache_only: - raise FileNotFoundError("'--singularity-cache' specified but no '$NXF_SINGULARITY_CACHEDIR' set!") + elif self.singularity_cache in ["amend", "copy"]: + raise FileNotFoundError("Singularity cache is required but no '$NXF_SINGULARITY_CACHEDIR' set!") return (out_path, cache_path) @@ -998,7 +1087,6 @@ def __init__( remote_url, revision, commit, - no_pull=False, hide_progress=False, in_cache=True, ): @@ -1028,6 +1116,7 @@ def __init__( self.commit = [] self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) self.retries = 0 # retries for setting up the locally cached repository + self.hide_progress = hide_progress self.setup_local_repo(remote_url, in_cache=in_cache) @@ -1089,7 +1178,7 @@ def setup_local_repo(self, remote, in_cache=True): rich.progress.BarColumn(bar_width=None), "[bold yellow]{task.fields[state]}", transient=True, - disable=os.environ.get("HIDE_PROGRESS", None) is not None, + disable=os.environ.get("HIDE_PROGRESS", None) is not None or self.hide_progress, ) with pbar: self.repo = git.Repo.clone_from( @@ -1112,7 +1201,7 @@ def setup_local_repo(self, remote, in_cache=True): rich.progress.BarColumn(bar_width=None), "[bold yellow]{task.fields[state]}", transient=True, - disable=os.environ.get("HIDE_PROGRESS", None) is not None, + disable=os.environ.get("HIDE_PROGRESS", None) is not None or self.hide_progress, ) with pbar: self.repo.remotes.origin.fetch( From d729bde26b96b0131048159ace4bc5e1867a30bf Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 26 Apr 2023 21:05:55 +0200 Subject: [PATCH 079/249] Readme updates for the new remote Singularity cache feature. --- README.md | 18 ++++++++++-------- nf_core/download.py | 22 +++++++++++----------- tests/test_cli.py | 6 ++++-- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 13d8b381a3..28c764a09a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ A python package with helper tools for the nf-core community. - [`nf-core list` - List available pipelines](#listing-pipelines) - [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) - [`nf-core download` - Download pipeline for offline use](#downloading-pipelines-for-offline-use) -- [`nf-core download --tower` - Download pipeline for Tower](#downloading-pipelines-for-tower) +- [`nf-core download --tower` - Adapting downloads to Nextflow Tower](#adapting-downloads-to-nextflow-tower) - [`nf-core licences` - List software licences in a pipeline](#pipeline-software-licences) - [`nf-core create` - Create a new pipeline with the nf-core template](#creating-a-new-pipeline) - [`nf-core lint` - Check pipeline code against nf-core guidelines](#linting-a-workflow) @@ -349,13 +349,13 @@ nextflow run /path/to/download/nf-core-rnaseq-dev/workflow/ --input mydata.csv - ### Downloaded nf-core configs The pipeline files are automatically updated (`params.custom_config_base` is set to `../configs`), so that the local copy of institutional configs are available when running the pipeline. -So using `-profile ` should work if available within [nf-core/configs](https://github.com/nf-core/configs). +So using `-profile ` should work if available within [nf-core/configs](https://github.com/nf-core/configs). This option is not available when downloading a pipeline for use with [Nextflow Tower](#adapting-downloads-to-nextflow-tower) because the application manages all configurations separately. ### Downloading singularity containers If you're using Singularity, the `nf-core download` command can also fetch the required Singularity container images for you. To do this, select `singularity` in the prompt or specify `--container singularity` in the command. -Your archive / target output directory will then include three folders: `workflow`, `configs` and also `singularity-containers`. +Your archive / target output directory will then also include a separate folder `singularity-containers`. The downloaded workflow files are again edited to add the following line to the end of the pipeline's `nextflow.config` file: @@ -373,11 +373,13 @@ We highly recommend setting the `$NXF_SINGULARITY_CACHEDIR` environment variable If found, the tool will fetch the Singularity images to this directory first before copying to the target output archive / directory. Any images previously fetched will be found there and copied directly - this includes images that may be shared with other pipelines or previous pipeline version downloads or download attempts. -If you are running the download on the same system where you will be running the pipeline (eg. a shared filesystem where Nextflow won't have an internet connection at a later date), you can choose to _only_ use the cache via a prompt or cli options `--singularity-cache-only` / `--singularity-cache-copy`. +If you are running the download on the same system where you will be running the pipeline (eg. a shared filesystem where Nextflow won't have an internet connection at a later date), you can choose to _only_ use the cache via a prompt or cli options `--singularity-cache amend`. This instructs `nf-core download` to fetch all Singularity images to the `$NXF_SINGULARITY_CACHEDIR` directory but does _not_ copy them to the workflow archive / directory. The workflow config file is _not_ edited. This means that when you later run the workflow, Nextflow will just use the cache folder directly. +If you are downloading a workflow for a different system, you can provide information about its image cache to `nf-core download`. To avoid unnecessary downloads, choose `--singularity-cache remote` and provide a list of already available images as plain text file to `--singularity-cache-index my_list_of_remotely_available_images.txt`. To generate this list on the remote system, run `find $NXF_SINGULARITY_CACHEDIR -name "*.img" > my_list_of_remotely_available_images.txt`. + #### How the Singularity image downloads work The Singularity image download finds containers using two methods: @@ -392,13 +394,13 @@ Where both are found, the download URL is preferred. Once a full list of containers is found, they are processed in the following order: -1. If the target image already exists, nothing is done (eg. with `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache-only` specified) -2. If found in `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache-only` is _not_ specified, they are copied to the output directory +1. If the target image already exists, nothing is done (eg. with `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache amend` specified) +2. If found in `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache copy` is specified, they are copied to the output directory 3. If they start with `http` they are downloaded directly within Python (default 4 at a time, you can customise this with `--parallel-downloads`) 4. If they look like a Docker image name, they are fetched using a `singularity pull` command - - This requires Singularity to be installed on the system and is substantially slower + - This requires Singularity/Apptainer to be installed on the system and is substantially slower -Note that compressing many GBs of binary files can be slow, so specifying `--compress none` is recommended when downloading Singularity images. +Note that compressing many GBs of binary files can be slow, so specifying `--compress none` is recommended when downloading Singularity images that are copied to the output directory. If the download speeds are much slower than your internet connection is capable of, you can set `--parallel-downloads` to a large number to download loads of images at once. diff --git a/nf_core/download.py b/nf_core/download.py index 59e7f47d3f..8297eb3f5c 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -141,11 +141,12 @@ def download_workflow(self): # Inclusion of configs is unnecessary for Tower. if not self.tower and self.include_configs is None: self.prompt_config_inclusion() + # If a remote cache is specified, it is safe to assume images should be downloaded. if not self.singularity_cache == "remote": self.prompt_container_download() - self.prompt_singularity_cachedir_creation() else: self.container = "singularity" + self.prompt_singularity_cachedir_creation() self.prompt_singularity_cachedir_utilization() self.prompt_singularity_cachedir_remote(retry=False) self.read_remote_containers() @@ -371,7 +372,7 @@ def prompt_config_inclusion(self): def prompt_container_download(self): """Prompt whether to download container images or not""" - if self.container is None: + if self.container is None and stderr.is_interactive: stderr.print("\nIn addition to the pipeline code, this tool can download software containers.") self.container = questionary.select( "Download software container images:", @@ -393,7 +394,8 @@ def prompt_singularity_cachedir_creation(self): if rich.prompt.Confirm.ask( "[blue bold]?[/] [bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" ): - self.singularity_cache == "amend" + if not self.singularity_cache_index: + self.singularity_cache == "amend" # retain "remote" choice. # Prompt user for a cache directory path cachedir_path = None while cachedir_path is None: @@ -419,7 +421,7 @@ def prompt_singularity_cachedir_creation(self): if bashrc_path: stderr.print( f"\nSo that [blue]$NXF_SINGULARITY_CACHEDIR[/] is always defined, you can add it to your [blue not bold]~/{os.path.basename(bashrc_path)}[/] file ." - "This will then be autmoatically set every time you open a new terminal. We can add the following line to this file for you: \n" + "This will then be automatically set every time you open a new terminal. We can add the following line to this file for you: \n" f'[blue]export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"[/]' ) append_to_file = rich.prompt.Confirm.ask( @@ -444,16 +446,18 @@ def prompt_singularity_cachedir_utilization(self): self.singularity_cache is None # no choice regarding singularity cache has been made. and self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None + and stderr.is_interactive ): stderr.print( "\nIf you are working on the same system where you will run Nextflow, you can amend the downloaded images to the ones in the" "[blue not bold]$NXF_SINGULARITY_CACHEDIR[/] folder, Nextflow will automatically find them." "However if you will transfer the downloaded files to a different system then they should be copied to the target folder." ) - self.singularity_cache = rich.prompt.Prompt.ask( - "[blue bold]?[/] [bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder or amend new images to the collection?[/]", + self.singularity_cache = questionary.select( + "[blue bold]?[/] [bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder or amend new images to the cache?[/]", choices=["amend", "copy"], - ) + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() def prompt_singularity_cachedir_remote(self, retry): """Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR""" @@ -463,10 +467,6 @@ def prompt_singularity_cachedir_remote(self, retry): and self.singularity_cache_index is None and stderr.is_interactive # Use rich auto-detection of interactive shells ): - stderr.print( - "\nNextflow and nf-core can use an environment variable called [blue]$NXF_SINGULARITY_CACHEDIR[/] that is a path to a directory where remote Singularity images are stored. " - "This allows downloaded images to be cached in a central location." - ) # Prompt user for a file listing the contents of the remote cache directory cachedir_index = None while cachedir_index is None: diff --git a/tests/test_cli.py b/tests/test_cli.py index 6f51fe1025..873b7d4b0c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -167,7 +167,8 @@ def test_cli_download(self, mock_dl): "force": None, "tower": None, "container": "singularity", - "singularity-cache-only": None, + "singularity-cache": "copy", + "singularity-cache-index": "/path/index.txt", "parallel-downloads": 2, } @@ -184,7 +185,8 @@ def test_cli_download(self, mock_dl): "force" in params, "tower" in params, params["container"], - "singularity-cache-only" in params, + params["singularity-cache"], + params["singularity-cache-index"], params["parallel-downloads"], ) From 0f58c29352c816056045450cac90ed7e8e6bfdc6 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 27 Apr 2023 11:49:21 +0200 Subject: [PATCH 080/249] Add interactive check in retry for parsing the index. --- nf_core/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index 8297eb3f5c..76a3f00054 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -512,7 +512,7 @@ def read_remote_containers(self): self.containers_remote = sorted(list(set(self.containers_remote))) except (FileNotFoundError, LookupError) as e: log.error(f"[red]Issue with reading the specified remote $NXF_SINGULARITY_CACHE index:[/]\n{e}\n") - if rich.prompt.Confirm.ask(f"[blue]Specify a new index file and try again?"): + if stderr.is_interactive and rich.prompt.Confirm.ask(f"[blue]Specify a new index file and try again?"): self.prompt_singularity_cachedir_remote(retry=True) else: log.info("Proceeding without consideration of the remote $NXF_SINGULARITY_CACHE index.") From 6294d74559cabebad29d6cb08d3aed6b4c57756b Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 27 Apr 2023 12:54:16 +0200 Subject: [PATCH 081/249] Incorporating some suggestions by @mashehu. --- nf_core/download.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 76a3f00054..b8ce5a1607 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -148,8 +148,7 @@ def download_workflow(self): self.container = "singularity" self.prompt_singularity_cachedir_creation() self.prompt_singularity_cachedir_utilization() - self.prompt_singularity_cachedir_remote(retry=False) - self.read_remote_containers() + self.prompt_singularity_cachedir_remote() # Nothing meaningful to compress here. if not self.tower: self.prompt_compression_type() @@ -177,8 +176,8 @@ def download_workflow(self): if not self.tower: # Only show entry, if option was prompted. summary_log.append(f"Include default institutional configuration: '{self.include_configs}'") - - summary_log.append(f"Enabled for seqeralabs® Nextflow Tower: '{self.tower}'") + else: + summary_log.append(f"Enabled for seqeralabs® Nextflow Tower: '{self.tower}'") # Check that the outdir doesn't already exist if os.path.exists(self.outdir): @@ -203,9 +202,9 @@ def download_workflow(self): if self.tower: self.download_workflow_tower() else: - self.download_workflow_classic() + self.download_workflow_static() - def download_workflow_classic(self): + def download_workflow_static(self): """Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner.""" # Download the centralised configs first @@ -290,7 +289,8 @@ def prompt_revision(self): Prompt for pipeline revision / branch Prompt user for revision tag if '--revision' was not set If --tower is specified, allow to select multiple revisions - Also the classic download allows for multiple revisions, but + Also the static download allows for multiple revisions, but + we do not prompt this option interactively. """ if not bool(self.revision): (choice, tag_set) = nf_core.utils.prompt_pipeline_release_branch( @@ -459,7 +459,7 @@ def prompt_singularity_cachedir_utilization(self): style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - def prompt_singularity_cachedir_remote(self, retry): + def prompt_singularity_cachedir_remote(self): """Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR""" if ( self.container == "singularity" @@ -485,8 +485,8 @@ def prompt_singularity_cachedir_remote(self, retry): cachedir_index = None if cachedir_index: self.singularity_cache_index = cachedir_index - if retry: # invoke parsing the file again. - self.read_remote_containers() + # in any case read the remote containers, even if no prompt was shown. + self.read_remote_containers() def read_remote_containers(self): """Reads the file specified as index for the remote Singularity cache dir""" From 8d327a46480dc55ea0e4fcf7e06d31d8b58c609f Mon Sep 17 00:00:00 2001 From: Matthias Zepper <6963520+MatthiasZepper@users.noreply.github.com> Date: Thu, 27 Apr 2023 12:58:46 +0200 Subject: [PATCH 082/249] Apply suggestions from code review @mashehu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/__main__.py | 4 ++-- nf_core/download.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 46e9ac0988..8b94e64715 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -213,14 +213,14 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all "-r", "--revision", multiple=True, - help="Pipeline release to download. Multiple invocations are possible.", + help="Pipeline release to download. Multiple invocations are possible, e.g. `-r 1.1 -r 1.2.", ) @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( "-x", "--compress", type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]), help="Archive compression type" ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") -@click.option("-t", "--tower", is_flag=True, default=False, help="Customize download for seqeralabs® Nextflow Tower") +@click.option("-t", "--tower", is_flag=True, default=False, help="Download for seqeralabs® Nextflow Tower") @click.option( "-c", "--container", type=click.Choice(["none", "singularity"]), help="Download software container images" ) diff --git a/nf_core/download.py b/nf_core/download.py index b8ce5a1607..8832eca5d3 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -242,7 +242,7 @@ def download_workflow_static(self): self.compress_download() def download_workflow_tower(self): - """Create a bare-cloned git repository of the workflow, such it can be launched with `tw launch` as file:/ pipeline""" + """Create a bare-cloned git repository of the workflow, so it can be launched with `tw launch` as file:/ pipeline""" log.info("Collecting workflow from GitHub") @@ -481,7 +481,7 @@ def prompt_singularity_cachedir_remote(self): self.singularity_cache_index = None self.singularity_cache = "copy" elif not os.access(cachedir_index, os.R_OK): - log.error(f"'{cachedir_index}' is not a valid, readable file.") + log.error(f"'{cachedir_index}' is not a readable file.") cachedir_index = None if cachedir_index: self.singularity_cache_index = cachedir_index From 340c5195067809b27d1f1da5273d9d2ca99bafc3 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 27 Apr 2023 20:12:15 +0200 Subject: [PATCH 083/249] Writing additional tests for the --tower download functionality. --- .../workflows/pytest-frozen-ubuntu-20.04.yml | 2 +- CHANGELOG.md | 1 + nf_core/__main__.py | 2 +- nf_core/download.py | 95 +++++++++++-------- tests/data/testdata_remote_containers.txt | 37 ++++++++ tests/test_download.py | 91 +++++++++++++++++- 6 files changed, 188 insertions(+), 40 deletions(-) create mode 100644 tests/data/testdata_remote_containers.txt diff --git a/.github/workflows/pytest-frozen-ubuntu-20.04.yml b/.github/workflows/pytest-frozen-ubuntu-20.04.yml index b015376633..5faf8ce605 100644 --- a/.github/workflows/pytest-frozen-ubuntu-20.04.yml +++ b/.github/workflows/pytest-frozen-ubuntu-20.04.yml @@ -15,7 +15,7 @@ concurrency: cancel-in-progress: true jobs: - pytest: + pytest-frozen: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index d880e0105d..513f1b014e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ ### Download - Introduce a `--tower` flag for `nf-core download` to obtain pipelines in an offline format suited for [seqeralabs® Nextflow Tower](https://cloud.tower.nf/) ([#2247](https://github.com/nf-core/tools/pull/2247)). +- Refactored the CLI for `--singularity-cache` in `nf-core download` from a flag to an argument. The prior options were renamed to `amend` (container images are only saved in the `$NXF_SINGULARITY_CACHEDIR`) and `copy` (a copy of the image is saved with the download). `remote` was newly introduced and allows to provide a table of contents of a remote cache via an additional argument `--singularity-cache-index` ([#2247](https://github.com/nf-core/tools/pull/2247)). ### Linting diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 8b94e64715..6d6ded471a 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -213,7 +213,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all "-r", "--revision", multiple=True, - help="Pipeline release to download. Multiple invocations are possible, e.g. `-r 1.1 -r 1.2.", + help="Pipeline release to download. Multiple invocations are possible, e.g. `-r 1.1 -r 1.2`", ) @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( diff --git a/nf_core/download.py b/nf_core/download.py index 8832eca5d3..db98b17f22 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -108,7 +108,7 @@ def __init__( self.force = force self.tower = tower self.include_configs = None - self.container = container + self.container = container if not singularity_cache_index else "singularity" self.singularity_cache = ( singularity_cache if not singularity_cache_index else "remote" ) # if a singularity_cache_index is given, use the file and overrule choice. @@ -157,7 +157,7 @@ def download_workflow(self): sys.exit(1) summary_log = [ - f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", + f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',...['+str(len(self.revision)-2)+' more revisions]...,'+self.revision[-1]}'", f"Pull containers: '{self.container}'", ] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: @@ -228,28 +228,29 @@ def download_workflow_static(self): # Collect all required singularity images if self.container == "singularity": - self.find_container_images(revision_dirname) + self.find_container_images(os.path.join(self.outdir, revision_dirname)) - try: - self.get_singularity_images() - except OSError as e: - log.critical(f"[red]{e}[/]") - sys.exit(1) + try: + self.get_singularity_images(current_revision=item[0]) + except OSError as e: + log.critical(f"[red]{e}[/]") + sys.exit(1) # Compress into an archive if self.compress_type is not None: log.info("Compressing output into archive") self.compress_download() - def download_workflow_tower(self): + def download_workflow_tower(self, location=None): """Create a bare-cloned git repository of the workflow, so it can be launched with `tw launch` as file:/ pipeline""" log.info("Collecting workflow from GitHub") self.workflow_repo = WorkflowRepo( - remote_url=f"git@github.com:{self.pipeline}.git", + remote_url=f"https://github.com/{self.pipeline}.git", revision=self.revision if self.revision else None, commit=self.wf_sha.values() if bool(self.wf_sha) else None, + location=location if location else None, # manual location is required for the tests to work in_cache=False, ) @@ -261,17 +262,17 @@ def download_workflow_tower(self): # extract the required containers if self.container == "singularity": - for commit in self.wf_sha.values(): + for revision, commit in self.wf_sha.items(): # Checkout the repo in the current revision self.workflow_repo.checkout(commit) # Collect all required singularity images self.find_container_images(self.workflow_repo.access()) - try: - self.get_singularity_images() - except OSError as e: - log.critical(f"[red]{e}[/]") - sys.exit(1) + try: + self.get_singularity_images(current_revision=revision) + except OSError as e: + log.critical(f"[red]{e}[/]") + sys.exit(1) # Justify why compression is skipped for Tower downloads (Prompt is not shown, but CLI argument could have been set) if self.compress_type is not None: @@ -412,30 +413,47 @@ def prompt_singularity_cachedir_creation(self): if cachedir_path: os.environ["NXF_SINGULARITY_CACHEDIR"] = cachedir_path - # Ask if user wants this set in their .bashrc - bashrc_path = os.path.expanduser("~/.bashrc") - if not os.path.isfile(bashrc_path): - bashrc_path = os.path.expanduser("~/.bash_profile") - if not os.path.isfile(bashrc_path): - bashrc_path = False - if bashrc_path: + """ + Optionally, create a permanent entry for the NXF_SINGULARITY_CACHEDIR in the terminal profile. + Currently support for bash and zsh. + ToDo: "sh", "bash", "dash", "ash","csh", "tcsh", "ksh", "zsh", "fish", "cmd", "powershell", "pwsh"? + """ + + if os.environ["SHELL"] == "/bin/bash": + shellprofile_path = os.path.expanduser("~/~/.bash_profile") + if not os.path.isfile(shellprofile_path): + shellprofile_path = os.path.expanduser("~/.bashrc") + if not os.path.isfile(shellprofile_path): + shellprofile_path = False + elif os.environ["SHELL"] == "/bin/zsh": + shellprofile_path = os.path.expanduser("~/.zprofile") + if not os.path.isfile(shellprofile_path): + shellprofile_path = os.path.expanduser("~/.zshenv") + if not os.path.isfile(shellprofile_path): + shellprofile_path = False + else: + shellprofile_path = os.path.expanduser("~/.profile") + if not os.path.isfile(shellprofile_path): + shellprofile_path = False + + if shellprofile_path: stderr.print( - f"\nSo that [blue]$NXF_SINGULARITY_CACHEDIR[/] is always defined, you can add it to your [blue not bold]~/{os.path.basename(bashrc_path)}[/] file ." + f"\nSo that [blue]$NXF_SINGULARITY_CACHEDIR[/] is always defined, you can add it to your [blue not bold]~/{os.path.basename(shellprofile_path)}[/] file ." "This will then be automatically set every time you open a new terminal. We can add the following line to this file for you: \n" f'[blue]export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"[/]' ) append_to_file = rich.prompt.Confirm.ask( - f"[blue bold]?[/] [bold]Add to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" + f"[blue bold]?[/] [bold]Add to [blue not bold]~/{os.path.basename(shellprofile_path)}[/] ?[/]" ) if append_to_file: - with open(os.path.expanduser(bashrc_path), "a") as f: + with open(os.path.expanduser(shellprofile_path), "a") as f: f.write( "\n\n#######################################\n" f"## Added by `nf-core download` v{nf_core.__version__} ##\n" + f'export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"' + "\n#######################################\n" ) - log.info(f"Successfully wrote to [blue]{bashrc_path}[/]") + log.info(f"Successfully wrote to [blue]{shellprofile_path}[/]") log.warning( "You will need reload your terminal after the download completes for this to take effect." ) @@ -620,7 +638,7 @@ def wf_use_local_configs(self, revision_dirname): with open(nfconfig_fn, "w") as nfconfig_fh: nfconfig_fh.write(nfconfig) - def find_container_images(self, revision_dirname): + def find_container_images(self, workflow_directory): """Find container image names for workflow. Starts by using `nextflow config` to pull out any process.container @@ -662,7 +680,7 @@ def find_container_images(self, revision_dirname): containers_raw = [] if not self.containers else self.containers # Use linting code to parse the pipeline nextflow config - self.nf_config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, revision_dirname)) + self.nf_config = nf_core.utils.fetch_wf_config(workflow_directory) # Find any config variables that look like a container for k, v in self.nf_config.items(): @@ -670,7 +688,7 @@ def find_container_images(self, revision_dirname): containers_raw.append(v.strip('"').strip("'")) # Recursive search through any DSL2 module files for container spec lines. - for subdir, _, files in os.walk(os.path.join(self.outdir, revision_dirname, "modules")): + for subdir, _, files in os.walk(os.path.join(workflow_directory, "modules")): for file in files: if file.endswith(".nf"): file_path = os.path.join(subdir, file) @@ -745,14 +763,14 @@ def find_container_images(self, revision_dirname): # Remove duplicates and sort self.containers = sorted(list(set(containers_raw))) - def get_singularity_images(self): + def get_singularity_images(self, current_revision=""): """Loop through container names and download Singularity images""" if len(self.containers) == 0: log.info("No container names found in workflow") else: log.info( - f"Found {len(self.containers)} container image{'s' if len(self.containers) > 1 else ''} in workflow." + f"Processing workflow revision {current_revision}, found {len(self.containers)} container image{'s' if len(self.containers) > 1 else ''} in total." ) with DownloadProgress() as progress: @@ -1087,6 +1105,7 @@ def __init__( remote_url, revision, commit, + location=None, hide_progress=False, in_cache=True, ): @@ -1118,7 +1137,7 @@ def __init__( self.retries = 0 # retries for setting up the locally cached repository self.hide_progress = hide_progress - self.setup_local_repo(remote_url, in_cache=in_cache) + self.setup_local_repo(remote=remote_url, location=location, in_cache=in_cache) # expose some instance attributes self.tags = self.repo.tags @@ -1155,7 +1174,7 @@ def retry_setup_local_repo(self, skip_confirm=False): else: raise LookupError("Exiting due to error with locally cached Git repository.") - def setup_local_repo(self, remote, in_cache=True): + def setup_local_repo(self, remote, location=None, in_cache=True): """ Sets up the local git repository. If the repository has been cloned previously, it returns a git.Repo object of that clone. Otherwise it tries to clone the repository from @@ -1163,13 +1182,15 @@ def setup_local_repo(self, remote, in_cache=True): Args: remote (str): git url of remote - commit (str): name of branch to checkout from (optional) - hide_progress (bool, optional): Whether to hide the progress bar. Defaults to False. + location (Path): location where the clone should be created/cached. in_cache (bool, optional): Whether to clone the repository from the cache. Defaults to False. Sets self.repo """ + if location: + self.local_repo_dir = os.path.join(location, self.fullname) + else: + self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname) - self.local_repo_dir = os.path.join(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname) try: if not os.path.exists(self.local_repo_dir): try: diff --git a/tests/data/testdata_remote_containers.txt b/tests/data/testdata_remote_containers.txt new file mode 100644 index 0000000000..93cf46f2f6 --- /dev/null +++ b/tests/data/testdata_remote_containers.txt @@ -0,0 +1,37 @@ +./depot.galaxyproject.org-singularity-bbmap-38.93--he522d1c_0.img +./depot.galaxyproject.org-singularity-bedtools-2.30.0--hc088bd4_0.img +./depot.galaxyproject.org-singularity-bioconductor-dupradar-1.18.0--r40_1.img +./depot.galaxyproject.org-singularity-bioconductor-summarizedexperiment-1.20.0--r40_0.img +./depot.galaxyproject.org-singularity-bioconductor-tximeta-1.8.0--r40_0.img +./depot.galaxyproject.org-singularity-fastqc-0.11.9--0.img +./depot.galaxyproject.org-singularity-gffread-0.12.1--h8b12597_0.img +./depot.galaxyproject.org-singularity-hisat2-2.2.1--h1b792b2_3.img +./depot.galaxyproject.org-singularity-mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2-59cdd445419f14abac76b31dd0d71217994cbcc9-0.img +./depot.galaxyproject.org-singularity-mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2-afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0.img +./depot.galaxyproject.org-singularity-mulled-v2-8849acf39a43cdd6c839a369a74c0adc823e2f91-ab110436faf952a33575c64dd74615a84011450b-0.img +./depot.galaxyproject.org-singularity-mulled-v2-a97e90b3b802d1da3d6958e0867610c718cb5eb1-0e773bb207600fcb4d38202226eb20a33c7909b6-0.img +./depot.galaxyproject.org-singularity-mulled-v2-a97e90b3b802d1da3d6958e0867610c718cb5eb1-38aed4501da19db366dc7c8d52d31d94e760cfaf-0.img +./depot.galaxyproject.org-singularity-mulled-v2-cf0123ef83b3c38c13e3b0696a3f285d3f20f15b-64aad4a4e144878400649e71f42105311be7ed87-0.img +./depot.galaxyproject.org-singularity-multiqc-1.11--pyhdfd78af_0.img +./depot.galaxyproject.org-singularity-multiqc-1.13--pyhdfd78af_0.img +./depot.galaxyproject.org-singularity-perl-5.26.2.img +./depot.galaxyproject.org-singularity-picard-2.26.10--hdfd78af_0.img +./depot.galaxyproject.org-singularity-picard-2.27.4--hdfd78af_0.img +./depot.galaxyproject.org-singularity-preseq-3.1.2--h445547b_2.img +./depot.galaxyproject.org-singularity-python-3.9--1.img +./depot.galaxyproject.org-singularity-qualimap-2.2.2d--1.img +./depot.galaxyproject.org-singularity-rseqc-3.0.1--py37h516909a_1.img +./depot.galaxyproject.org-singularity-salmon-1.5.2--h84f40af_0.img +./depot.galaxyproject.org-singularity-samtools-1.15.1--h1170115_0.img +./depot.galaxyproject.org-singularity-sortmerna-4.3.4--h9ee0642_0.img +./depot.galaxyproject.org-singularity-stringtie-2.2.1--hecb563c_2.img +./depot.galaxyproject.org-singularity-subread-2.0.1--hed695b0_0.img +./depot.galaxyproject.org-singularity-trim-galore-0.6.7--hdfd78af_0.img +./depot.galaxyproject.org-singularity-ubuntu-20.04.img +./depot.galaxyproject.org-singularity-ucsc-bedclip-377--h0b8a92a_2.img +./depot.galaxyproject.org-singularity-ucsc-bedgraphtobigwig-377--h446ed27_1.img +./depot.galaxyproject.org-singularity-umi_tools-1.1.2--py38h4a8c8d9_0.img +These entries should not be used: +On October 5, 2011, the 224-meter containership MV Rena struck a reef close to New Zealand’s coast and broke apart. That spells disaster, no? +MV Rena + diff --git a/tests/test_download.py b/tests/test_download.py index d1a770a630..41fb9c625f 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -3,16 +3,20 @@ import hashlib import os +import re import shutil import tempfile import unittest +from pathlib import Path from unittest import mock import pytest import nf_core.create import nf_core.utils -from nf_core.download import DownloadWorkflow +from nf_core.download import DownloadWorkflow, WorkflowRepo +from nf_core.synced_repo import SyncedRepo +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR from .utils import with_temporary_file, with_temporary_folder @@ -169,6 +173,32 @@ def test_singularity_pull_image_singularity_not_installed(self, tmp_dir, mock_ri with pytest.raises(OSError): download_obj.singularity_pull_image("a-container", tmp_dir, None, mock_rich_progress) + # + # Test for '--singularity-cache remote --singularity-cache-index'. Provide a list of containers already available in a remote location. + # + @with_temporary_folder + def test_remote_container_functionality(self, tmp_dir): + os.environ["NXF_SINGULARITY_CACHEDIR"] = "foo" + + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + outdir=os.path.join(tmp_dir, "new"), + revision="3.9", + compress_type="none", + singularity_cache_index=Path(__file__).resolve().parent / "data/testdata_remote_containers.txt", + ) + + download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. + + # test if settings are changed accordingly. + assert download_obj.singularity_cache == "remote" and download_obj.container == "singularity" + assert isinstance(download_obj.containers_remote, list) and len(download_obj.containers_remote) == 0 + # read in the file + download_obj.read_remote_containers() + assert len(download_obj.containers_remote) == 33 + assert "depot.galaxyproject.org-singularity-salmon-1.5.2--h84f40af_0.img" in download_obj.containers_remote + assert "MV Rena" not in download_obj.containers_remote # decoy in test file + # # Tests for the main entry method 'download_workflow' # @@ -184,6 +214,65 @@ def test_download_workflow_with_success(self, tmp_dir, mock_download_image, mock container="singularity", revision="1.6", compress_type="none", + singularity_cache="copy", ) + download_obj.include_configs = True # suppress prompt, because stderr.is_interactive doesn't. download_obj.download_workflow() + + # + # Test Download for Tower + # + @with_temporary_folder + def test_download_workflow_for_tower(self, tmp_dir): + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.7", "3.9"), + compress_type="none", + tower=True, + ) + + download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. + + assert isinstance(download_obj.revision, list) and len(download_obj.revision) == 2 + assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 0 + assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 + + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(download_obj.pipeline, wfs) + + download_obj.get_revision_hash() + + # download_obj.wf_download_url is not set for tower downloads, but the sha values are + assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 2 + assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 + + # The outdir for multiple revisions is the pipeline name and date: e.g. nf-core-rnaseq_2023-04-27_18-54 + assert bool(re.search(r"nf-core-rnaseq_\d{4}-\d{2}-\d{1,2}_\d{1,2}-\d{1,2}", download_obj.outdir, re.S)) + + download_obj.output_filename = f"{download_obj.outdir}.git" + download_obj.download_workflow_tower(location=tmp_dir) + + assert download_obj.workflow_repo + assert isinstance(download_obj.workflow_repo, WorkflowRepo) + assert issubclass(type(download_obj.workflow_repo), SyncedRepo) + # corroborate that the other revisions are inaccessible to the user. + assert len(download_obj.workflow_repo.tags) == len(download_obj.revision) + + # manually test container image detection for 3.7 revision + download_obj.workflow_repo.checkout(download_obj.wf_sha["3.7"]) + assert isinstance(download_obj.containers, list) and len(download_obj.containers) == 0 + download_obj.find_container_images(download_obj.workflow_repo.access()) + assert len(download_obj.containers) == 30 # 30 containers for 3.7 + assert ( + "https://depot.galaxyproject.org/singularity/bbmap:38.93--he522d1c_0" in download_obj.containers + ) # direct definition + assert ( + "https://depot.galaxyproject.org/singularity/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:59cdd445419f14abac76b31dd0d71217994cbcc9-0" + in download_obj.containers + ) # indirect definition via $container variable. From f599237c21557e2692da7e05c232bfe4f290c5a0 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 2 May 2023 15:52:01 +0200 Subject: [PATCH 084/249] Move alterations from Version 2.8 (which this PR didn't make anymore) to Version 2.9dev. --- CHANGELOG.md | 9 ++++----- README.md | 7 ++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 513f1b014e..fde3775e70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ - Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286) - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) +### Download + +- Introduce a `--tower` flag for `nf-core download` to obtain pipelines in an offline format suited for [seqeralabs® Nextflow Tower](https://cloud.tower.nf/) ([#2247](https://github.com/nf-core/tools/pull/2247)). +- Refactored the CLI for `--singularity-cache` in `nf-core download` from a flag to an argument. The prior options were renamed to `amend` (container images are only saved in the `$NXF_SINGULARITY_CACHEDIR`) and `copy` (a copy of the image is saved with the download). `remote` was newly introduced and allows to provide a table of contents of a remote cache via an additional argument `--singularity-cache-index` ([#2247](https://github.com/nf-core/tools/pull/2247)). ### Linting @@ -49,11 +53,6 @@ - Removed `quay.io` from all module Docker container references as this is now supplied at pipeline level. ([#2249](https://github.com/nf-core/tools/pull/2249)) - Remove `CITATION.cff` file from pipeline template, to avoid that pipeline Zenodo entries reference the nf-core publication instead of the pipeline ([#2059](https://github.com/nf-core/tools/pull/2059)). -### Download - -- Introduce a `--tower` flag for `nf-core download` to obtain pipelines in an offline format suited for [seqeralabs® Nextflow Tower](https://cloud.tower.nf/) ([#2247](https://github.com/nf-core/tools/pull/2247)). -- Refactored the CLI for `--singularity-cache` in `nf-core download` from a flag to an argument. The prior options were renamed to `amend` (container images are only saved in the `$NXF_SINGULARITY_CACHEDIR`) and `copy` (a copy of the image is saved with the download). `remote` was newly introduced and allows to provide a table of contents of a remote cache via an additional argument `--singularity-cache-index` ([#2247](https://github.com/nf-core/tools/pull/2247)). - ### Linting - Update modules lint test to fail if enable_conda is found ([#2213](https://github.com/nf-core/tools/pull/2213)) diff --git a/README.md b/README.md index 28c764a09a..06cae66c8e 100644 --- a/README.md +++ b/README.md @@ -373,12 +373,9 @@ We highly recommend setting the `$NXF_SINGULARITY_CACHEDIR` environment variable If found, the tool will fetch the Singularity images to this directory first before copying to the target output archive / directory. Any images previously fetched will be found there and copied directly - this includes images that may be shared with other pipelines or previous pipeline version downloads or download attempts. -If you are running the download on the same system where you will be running the pipeline (eg. a shared filesystem where Nextflow won't have an internet connection at a later date), you can choose to _only_ use the cache via a prompt or cli options `--singularity-cache amend`. +If you are running the download on the same system where you will be running the pipeline (eg. a shared filesystem where Nextflow won't have an internet connection at a later date), you can choose to _only_ use the cache via a prompt or cli options `--singularity-cache amend`. This instructs `nf-core download` to fetch all Singularity images to the `$NXF_SINGULARITY_CACHEDIR` directory but does _not_ copy them to the workflow archive / directory. The workflow config file is _not_ edited. This means that when you later run the workflow, Nextflow will just use the cache folder directly. -This instructs `nf-core download` to fetch all Singularity images to the `$NXF_SINGULARITY_CACHEDIR` directory but does _not_ copy them to the workflow archive / directory. -The workflow config file is _not_ edited. This means that when you later run the workflow, Nextflow will just use the cache folder directly. - -If you are downloading a workflow for a different system, you can provide information about its image cache to `nf-core download`. To avoid unnecessary downloads, choose `--singularity-cache remote` and provide a list of already available images as plain text file to `--singularity-cache-index my_list_of_remotely_available_images.txt`. To generate this list on the remote system, run `find $NXF_SINGULARITY_CACHEDIR -name "*.img" > my_list_of_remotely_available_images.txt`. +If you are downloading a workflow for a different system, you can provide information about its image cache to `nf-core download`. To avoid unnecessary container image downloads, choose `--singularity-cache remote` and provide a list of already available images as plain text file to `--singularity-cache-index my_list_of_remotely_available_images.txt`. To generate this list on the remote system, run `find $NXF_SINGULARITY_CACHEDIR -name "*.img" > my_list_of_remotely_available_images.txt`. The tool will then only download and copy images into your output directory, which are missing on the remote system. #### How the Singularity image downloads work From 2518a4bfda260064fae4b681777822e89d9e0586 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 2 May 2023 16:12:54 +0200 Subject: [PATCH 085/249] Adding the info about remote containers to the summary log rather than showing it separately. --- nf_core/download.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index db98b17f22..cfccf2a235 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -162,6 +162,10 @@ def download_workflow(self): ] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: summary_log.append(f"Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {os.environ['NXF_SINGULARITY_CACHEDIR']}") + if self.containers_remote: + summary_log.append( + f"Successfully read {len(self.containers_remote)} containers from the remote '[blue]$NXF_SINGULARITY_CACHEDIR[/]' contents." + ) # Set an output filename now that we have the outdir if self.tower: @@ -523,10 +527,6 @@ def read_remote_containers(self): self.containers_remote.append(match.group(0)) if n_total_images == 0: raise LookupError("Could not find valid container names in the index file.") - else: - log.info( - f"Successfully read {n_total_images} containers from the remote $NXF_SINGULARITY_CACHE contents." - ) self.containers_remote = sorted(list(set(self.containers_remote))) except (FileNotFoundError, LookupError) as e: log.error(f"[red]Issue with reading the specified remote $NXF_SINGULARITY_CACHE index:[/]\n{e}\n") @@ -825,7 +825,9 @@ def get_singularity_images(self, current_revision=""): if containers_exist: if self.singularity_cache_index is not None: - log.info(f"{len(containers_exist)} are already cached remotely and won't be retrieved.") + log.info( + f"{len(containers_exist)} containers are already cached remotely and won't be retrieved." + ) # Go through each method of fetching containers in order for container in containers_exist: progress.update(task, description="Image file exists at destination") From 4f390be96a7fc41f01443412fbb065933569336c Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 5 May 2023 11:48:07 +0200 Subject: [PATCH 086/249] Moved the notification about remote containers to summary_log. --- nf_core/download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index cfccf2a235..8a65252e3b 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -161,10 +161,10 @@ def download_workflow(self): f"Pull containers: '{self.container}'", ] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: - summary_log.append(f"Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {os.environ['NXF_SINGULARITY_CACHEDIR']}") + summary_log.append(f"Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {os.environ['NXF_SINGULARITY_CACHEDIR']}'") if self.containers_remote: summary_log.append( - f"Successfully read {len(self.containers_remote)} containers from the remote '[blue]$NXF_SINGULARITY_CACHEDIR[/]' contents." + f"Successfully read {len(self.containers_remote)} containers from the remote '$NXF_SINGULARITY_CACHEDIR' contents." ) # Set an output filename now that we have the outdir From f8e50684fd4eb5138205d2682ab5b2eaf1f3a3c6 Mon Sep 17 00:00:00 2001 From: Matthias Zepper <6963520+MatthiasZepper@users.noreply.github.com> Date: Tue, 9 May 2023 15:33:40 +0200 Subject: [PATCH 087/249] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/download.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 8a65252e3b..8705add039 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -157,7 +157,7 @@ def download_workflow(self): sys.exit(1) summary_log = [ - f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',...['+str(len(self.revision)-2)+' more revisions]...,'+self.revision[-1]}'", + f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',...,['+str(len(self.revision)-2)+' more revisions],...,'+self.revision[-1]}'", f"Pull containers: '{self.container}'", ] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: @@ -420,7 +420,7 @@ def prompt_singularity_cachedir_creation(self): """ Optionally, create a permanent entry for the NXF_SINGULARITY_CACHEDIR in the terminal profile. Currently support for bash and zsh. - ToDo: "sh", "bash", "dash", "ash","csh", "tcsh", "ksh", "zsh", "fish", "cmd", "powershell", "pwsh"? + ToDo: "sh", "dash", "ash","csh", "tcsh", "ksh", "fish", "cmd", "powershell", "pwsh"? """ if os.environ["SHELL"] == "/bin/bash": @@ -476,7 +476,7 @@ def prompt_singularity_cachedir_utilization(self): "However if you will transfer the downloaded files to a different system then they should be copied to the target folder." ) self.singularity_cache = questionary.select( - "[blue bold]?[/] [bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder or amend new images to the cache?[/]", + "Copy singularity images from $NXF_SINGULARITY_CACHEDIR to the target folder or amend new images to the cache?", choices=["amend", "copy"], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() From e5128786de1dcabc6909ea0c0e36406812290298 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 9 May 2023 15:46:26 +0200 Subject: [PATCH 088/249] Fixes suggested by @mirpedrol during review. Thanks! --- README.md | 3 +-- nf_core/download.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 06cae66c8e..dacb50ebc4 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,7 @@ A python package with helper tools for the nf-core community. - [`nf-core` tools update](#update-tools) - [`nf-core list` - List available pipelines](#listing-pipelines) - [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) -- [`nf-core download` - Download pipeline for offline use](#downloading-pipelines-for-offline-use) -- [`nf-core download --tower` - Adapting downloads to Nextflow Tower](#adapting-downloads-to-nextflow-tower) +- [`nf-core download` - Download a pipeline for offline use](#downloading-pipelines-for-offline-use) - [`nf-core licences` - List software licences in a pipeline](#pipeline-software-licences) - [`nf-core create` - Create a new pipeline with the nf-core template](#creating-a-new-pipeline) - [`nf-core lint` - Check pipeline code against nf-core guidelines](#linting-a-workflow) diff --git a/nf_core/download.py b/nf_core/download.py index 8705add039..5d214d2aaf 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -423,13 +423,13 @@ def prompt_singularity_cachedir_creation(self): ToDo: "sh", "dash", "ash","csh", "tcsh", "ksh", "fish", "cmd", "powershell", "pwsh"? """ - if os.environ["SHELL"] == "/bin/bash": + if os.getenv("SHELL", "") == "/bin/bash": shellprofile_path = os.path.expanduser("~/~/.bash_profile") if not os.path.isfile(shellprofile_path): shellprofile_path = os.path.expanduser("~/.bashrc") if not os.path.isfile(shellprofile_path): shellprofile_path = False - elif os.environ["SHELL"] == "/bin/zsh": + elif os.getenv("SHELL", "") == "/bin/zsh": shellprofile_path = os.path.expanduser("~/.zprofile") if not os.path.isfile(shellprofile_path): shellprofile_path = os.path.expanduser("~/.zshenv") From 315b9a3536a1d7cdd661bafc3e7f0f1cc63051a7 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 9 May 2023 17:00:47 +0200 Subject: [PATCH 089/249] @mashehu suggested that downloading the containers should not be optional for Tower downloads. Given that there is the option to provide the list of remote containers to skip their download, I agree that this is reasonable. --- CHANGELOG.md | 1 + nf_core/download.py | 12 ++++++------ nf_core/modules/lint/main_nf.py | 2 +- tests/test_download.py | 8 +++++--- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fde3775e70..89d3fe8f8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286) - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) + ### Download - Introduce a `--tower` flag for `nf-core download` to obtain pipelines in an offline format suited for [seqeralabs® Nextflow Tower](https://cloud.tower.nf/) ([#2247](https://github.com/nf-core/tools/pull/2247)). diff --git a/nf_core/download.py b/nf_core/download.py index 5d214d2aaf..274132f02b 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -108,10 +108,10 @@ def __init__( self.force = force self.tower = tower self.include_configs = None - self.container = container if not singularity_cache_index else "singularity" - self.singularity_cache = ( - singularity_cache if not singularity_cache_index else "remote" - ) # if a singularity_cache_index is given, use the file and overrule choice. + # force download of containers if a cache index is given or download is meant to be used for Tower. + self.container = "singularity" if singularity_cache_index or bool(tower) else container + # if a singularity_cache_index is given, use the file and overrule choice. + self.singularity_cache = "remote" if singularity_cache_index else singularity_cache self.singularity_cache_index = singularity_cache_index self.parallel_downloads = parallel_downloads @@ -377,7 +377,7 @@ def prompt_config_inclusion(self): def prompt_container_download(self): """Prompt whether to download container images or not""" - if self.container is None and stderr.is_interactive: + if self.container is None and stderr.is_interactive and not self.tower: stderr.print("\nIn addition to the pipeline code, this tool can download software containers.") self.container = questionary.select( "Download software container images:", @@ -722,7 +722,7 @@ def find_container_images(self, workflow_directory): Therefore, we need to repeat the search over the contents, extract the variable name, and use it inside a new regex. To get the variable name ( ${container_id} in above example ), we match the literal word "container" and use lookbehind (reset the match). - Then we skip [^\${}]+ everything that is not $ or curly braces. The next capture group is + Then we skip [^${}]+ everything that is not $ or curly braces. The next capture group is ${ followed by any characters that are not curly braces [^{}]+ and ended by a closing curly brace (}), but only if it's not followed by any other curly braces (?![^{]*}). The latter ensures we capture the innermost variable name. diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 8150e7e839..31b8adca3a 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -283,7 +283,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = NoneD if l.startswith("quay.io/"): - l_stripped = re.sub("\W+$", "", l) + l_stripped = re.sub(r"\W+$", "", l) self.failed.append( ( "container_links", diff --git a/tests/test_download.py b/tests/test_download.py index 41fb9c625f..aa2e959f3d 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -190,7 +190,7 @@ def test_remote_container_functionality(self, tmp_dir): download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. - # test if settings are changed accordingly. + # test if the settings are changed to mandatory defaults, if an external cache index is used. assert download_obj.singularity_cache == "remote" and download_obj.container == "singularity" assert isinstance(download_obj.containers_remote, list) and len(download_obj.containers_remote) == 0 # read in the file @@ -264,9 +264,11 @@ def test_download_workflow_for_tower(self, tmp_dir): # corroborate that the other revisions are inaccessible to the user. assert len(download_obj.workflow_repo.tags) == len(download_obj.revision) - # manually test container image detection for 3.7 revision + # download_obj.download_workflow_tower(location=tmp_dir) will run container image detection for all requested revisions + assert isinstance(download_obj.containers, list) and len(download_obj.containers) == 33 + # manually test container image detection for 3.7 revision only + download_obj.containers = [] # empty container list for the test download_obj.workflow_repo.checkout(download_obj.wf_sha["3.7"]) - assert isinstance(download_obj.containers, list) and len(download_obj.containers) == 0 download_obj.find_container_images(download_obj.workflow_repo.access()) assert len(download_obj.containers) == 30 # 30 containers for 3.7 assert ( From 2ee34a213fb0f7787fea90effbbabd3687dc1391 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 26 May 2023 11:28:51 +0200 Subject: [PATCH 090/249] avoid a blocking request call on first cli call --- nf_core/utils.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 36c39db50a..4e85d4eba5 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1,6 +1,7 @@ """ Common utility functions for the nf-core python package. """ +import concurrent.futures import datetime import errno import hashlib @@ -58,6 +59,12 @@ NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME"), ".config")), "nfcore") +def fetch_remote_version(source_url): + response = requests.get(source_url, timeout=3) + remote_version = re.sub(r"[^0-9\.]", "", response.text) + return remote_version + + def check_if_outdated(current_version=None, remote_version=None, source_url="https://nf-co.re/tools_version"): """ Check if the current version of nf-core is outdated @@ -72,12 +79,16 @@ def check_if_outdated(current_version=None, remote_version=None, source_url="htt # Build the URL to check against source_url = os.environ.get("NFCORE_VERSION_URL", source_url) source_url = f"{source_url}?v={current_version}" - # Fetch and clean up the remote version - if remote_version is None: - response = requests.get(source_url, timeout=3) - remote_version = re.sub(r"[^0-9\.]", "", response.text) - # Check if we have an available update - is_outdated = Version(remote_version) > Version(current_version) + + # Fetch the remote version asynchronously + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(fetch_remote_version, source_url) + # Retrieve the remote version in the background + remote_version = future.result() + is_outdated = False + if remote_version is not None: + # Check if we have an available update + is_outdated = Version(remote_version) > Version(current_version) return (is_outdated, current_version, remote_version) From 766b87a08062219d656a2fb4fa9e1e649a538483 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 26 May 2023 11:48:01 +0200 Subject: [PATCH 091/249] improved version of the async call --- nf_core/utils.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 7774265bd0..28a6e3f54a 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -79,16 +79,18 @@ def check_if_outdated(current_version=None, remote_version=None, source_url="htt # Build the URL to check against source_url = os.environ.get("NFCORE_VERSION_URL", source_url) source_url = f"{source_url}?v={current_version}" - - # Fetch the remote version asynchronously - with concurrent.futures.ThreadPoolExecutor() as executor: - future = executor.submit(fetch_remote_version, source_url) - # Retrieve the remote version in the background - remote_version = future.result() + # check if we have a newer version without blocking the rest of the script is_outdated = False + remote_version = None + try: + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(fetch_remote_version, source_url) + remote_version = future.result() + except Exception as e: + log.debug(f"Could not check for nf-core updates: {e}") if remote_version is not None: - # Check if we have an available update - is_outdated = Version(remote_version) > Version(current_version) + if Version(remote_version) > Version(current_version): + is_outdated = True return (is_outdated, current_version, remote_version) From 397ddec0440cbb9a6f029d771bae6b7b76ae668d Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 26 May 2023 12:16:18 +0200 Subject: [PATCH 092/249] add manual overwrite option for tests --- nf_core/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 28a6e3f54a..3cd09397e3 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -81,13 +81,13 @@ def check_if_outdated(current_version=None, remote_version=None, source_url="htt source_url = f"{source_url}?v={current_version}" # check if we have a newer version without blocking the rest of the script is_outdated = False - remote_version = None - try: - with concurrent.futures.ThreadPoolExecutor() as executor: - future = executor.submit(fetch_remote_version, source_url) - remote_version = future.result() - except Exception as e: - log.debug(f"Could not check for nf-core updates: {e}") + if remote_version is None: # we set it manually for tests + try: + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(fetch_remote_version, source_url) + remote_version = future.result() + except Exception as e: + log.debug(f"Could not check for nf-core updates: {e}") if remote_version is not None: if Version(remote_version) > Version(current_version): is_outdated = True From 6a806ee322e880db106e3fce8434d34527afca6b Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 26 May 2023 23:52:43 +0200 Subject: [PATCH 093/249] Bugfix: WorkflowRepo.tidy_tags() did indeed only tidy tags. However, revisions may also be branches. Therefore, I rewrote this function to account for revisions that are not releases. --- nf_core/download.py | 51 +++++++++++++++++++++++++++++++----------- nf_core/synced_repo.py | 2 +- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 274132f02b..70f61f35a4 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -259,7 +259,7 @@ def download_workflow_tower(self, location=None): ) # Remove tags for those revisions that had not been selected - self.workflow_repo.tidy_tags() + self.workflow_repo.tidy_tags_and_branches() # create a bare clone of the modified repository needed for Tower self.workflow_repo.bare_clone(os.path.join(self.outdir, self.output_filename)) @@ -1157,6 +1157,9 @@ def access(self): def checkout(self, commit): return super().checkout(commit) + def get_remote_branches(self, remote_url): + return super().get_remote_branches(remote_url) + def retry_setup_local_repo(self, skip_confirm=False): self.retries += 1 if skip_confirm or rich.prompt.Confirm.ask( @@ -1236,29 +1239,51 @@ def setup_local_repo(self, remote, location=None, in_cache=True): log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") self.retry_setup_local_repo() - def tidy_tags(self): + def tidy_tags_and_branches(self): """ - Function to delete all tags that point to revisions that are not of interest to the downloader. - This allows a clutter-free experience in Tower. The commits are evidently still available. + Function to delete all tags and branches that are not of interest to the downloader. + This allows a clutter-free experience in Tower. The untagged commits are evidently still available. However, due to local caching, the downloader might also want access to revisions that had been deleted before. In that case, don't bother with re-adding the tags and rather download anew from Github. """ if self.revision and self.repo and self.repo.tags: - desired_tags = self.revision.copy() + # create a set to keep track of the revisions to process & check + desired_revisions = set(self.revision) + + # determine what needs pruning + tags_to_remove = {tag for tag in self.repo.tags if tag.name not in desired_revisions} + heads_to_remove = {head for head in self.repo.heads if head.name not in desired_revisions} + try: - for tag in self.repo.tags: - if tag.name not in self.revision: - self.repo.delete_tag(tag) - else: - desired_tags.remove(tag.name) + # delete unwanted tags from repository + for tag in tags_to_remove: + self.repo.delete_tag(tag) self.tags = self.repo.tags - if len(desired_tags) > 0: + + # switch to a revision that should be kept, because deleting heads fails, if they are checked out (e.g. "master") + self.checkout(self.revision[0]) + + # delete unwanted heads/branches from repository + for head in heads_to_remove: + self.repo.delete_head(head) + + # ensure all desired branches are available + for revision in desired_revisions: + self.checkout(revision) + self.heads = self.repo.heads + + # get all tags and available remote_branches + completed_revisions = {revision.name for revision in self.repo.heads + self.repo.tags} + + # verify that all requested revisions are available. + # a local cache might lack revisions that were deleted during a less comprehensive previous download. + if bool(desired_revisions - completed_revisions): log.info( - f"Locally cached version of the pipeline lacks selected revisions {', '.join(desired_tags)}. Downloading anew from GitHub..." + f"Locally cached version of the pipeline lacks selected revisions {', '.join(desired_revisions - completed_revisions)}. Downloading anew from GitHub..." ) self.retry_setup_local_repo(skip_confirm=True) - self.tidy_tags() + self.tidy_tags_and_branches() except (GitCommandError, InvalidGitRepositoryError) as e: log.error(f"[red]Adapting your pipeline download unfortunately failed:[/]\n{e}\n") self.retry_setup_local_repo(skip_confirm=True) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 4bbd4f8443..f78142c031 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -196,7 +196,7 @@ def branch_exists(self): def verify_branch(self): """ - Verifies the active branch conforms do the correct directory structure + Verifies the active branch conforms to the correct directory structure """ dir_names = os.listdir(self.local_repo_dir) if "modules" not in dir_names: From 822e14fcdec49df45f8d5d7ecb33739270d4d53c Mon Sep 17 00:00:00 2001 From: SusiJo Date: Tue, 30 May 2023 10:18:09 +0200 Subject: [PATCH 094/249] rm outdated Dockerfile --- .github/CONTRIBUTING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index fc73294b5a..a90c06f9a9 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -142,4 +142,3 @@ To get started: Devcontainer specs: - [DevContainer config](.devcontainer/devcontainer.json) -- [Dockerfile](.devcontainer/Dockerfile) From a70a3fe47ceb5f6884810efa83ede6b77f3bd8ab Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 1 Jun 2023 13:45:37 +0200 Subject: [PATCH 095/249] add version and subpath to multiqc report comment link --- nf_core/pipeline-template/assets/multiqc_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index 440b0b9a3a..c3cf9647db 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -1,7 +1,7 @@ report_comment: > This report has been generated by the {{ name }} analysis pipeline.{% if branded %} For information about how to interpret these results, please see the - documentation.{% endif %} + documentation.{% endif %} report_section_order: "{{ name_noslash }}-methods-description": order: -1000 From f016eae974ed18e7b0d417098a96c61b90e907f7 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 1 Jun 2023 13:47:36 +0200 Subject: [PATCH 096/249] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b79fd410dc..7b99dafded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286) - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) +- Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) ### Linting From 63e9b8bf656e08f9d446eb46cd18fd3b8d5f9fcc Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 1 Jun 2023 13:49:12 +0200 Subject: [PATCH 097/249] also add version to main link --- nf_core/pipeline-template/assets/multiqc_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index c3cf9647db..570ed3d8e5 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -1,5 +1,5 @@ report_comment: > - This report has been generated by the {{ name }} + This report has been generated by the {{ name }} analysis pipeline.{% if branded %} For information about how to interpret these results, please see the documentation.{% endif %} report_section_order: From df9dee576a7eded130361bae761249e21fc5985c Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 1 Jun 2023 14:03:31 +0200 Subject: [PATCH 098/249] fix linting test --- nf_core/lint/multiqc_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index 3378efce5f..9eff60091f 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -71,12 +71,13 @@ def multiqc_config(self): if "report_comment" not in ignore_configs: # Check that the minimum plugins exist and are coming first in the summary try: + version = self.nf_config.get("manifest.version", "").strip(" '\"") if "report_comment" not in mqc_yml: raise AssertionError() if mqc_yml["report_comment"].strip() != ( - f'This report has been generated by the nf-core/{self.pipeline_name} analysis pipeline. For information about how to ' - f'interpret these results, please see the documentation.' ): raise AssertionError() From 36c9284202c4ecafaa25e31e4b7e1aa72f8843dc Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 1 Jun 2023 14:27:08 +0200 Subject: [PATCH 099/249] update multiqc_config during version bump --- nf_core/bump_version.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 129016fa38..2ab75f26bc 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -44,6 +44,17 @@ def bump_pipeline_version(pipeline_obj, new_version): ) ], ) + # multiqc_config.yaml + update_file_version( + "multiqc_config.yaml", + pipeline_obj, + [ + ( + rf"version\s*=\s*[\'\"]?{re.escape(current_version)}[\'\"]?", + f"version = '{new_version}'", + ) + ], + ) def bump_nextflow_version(pipeline_obj, new_version): From 531f9771665b73c913716a704c058e53cfcfceb5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 1 Jun 2023 15:07:43 +0200 Subject: [PATCH 100/249] bump version in multiqc_config correctly --- nf_core/bump_version.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 2ab75f26bc..b462ee1377 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -3,8 +3,8 @@ """ import logging -import os import re +from pathlib import Path import rich.console @@ -46,12 +46,12 @@ def bump_pipeline_version(pipeline_obj, new_version): ) # multiqc_config.yaml update_file_version( - "multiqc_config.yaml", + Path("assets", "multiqc_config.yml"), pipeline_obj, [ ( - rf"version\s*=\s*[\'\"]?{re.escape(current_version)}[\'\"]?", - f"version = '{new_version}'", + rf"{re.escape(current_version)}", + f"{new_version}", ) ], ) @@ -88,7 +88,7 @@ def bump_nextflow_version(pipeline_obj, new_version): # .github/workflows/ci.yml - Nextflow version matrix update_file_version( - os.path.join(".github", "workflows", "ci.yml"), + Path(".github", "workflows", "ci.yml"), pipeline_obj, [ ( From 6bbbd4a298d6a7e665a50f3c4f93c4542c193de5 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 1 Jun 2023 20:34:26 +0200 Subject: [PATCH 101/249] Small bugfixes. --- nf_core/download.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 70f61f35a4..0b32ba115e 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -157,7 +157,7 @@ def download_workflow(self): sys.exit(1) summary_log = [ - f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',...,['+str(len(self.revision)-2)+' more revisions],...,'+self.revision[-1]}'", + f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", f"Pull containers: '{self.container}'", ] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: @@ -531,7 +531,7 @@ def read_remote_containers(self): except (FileNotFoundError, LookupError) as e: log.error(f"[red]Issue with reading the specified remote $NXF_SINGULARITY_CACHE index:[/]\n{e}\n") if stderr.is_interactive and rich.prompt.Confirm.ask(f"[blue]Specify a new index file and try again?"): - self.prompt_singularity_cachedir_remote(retry=True) + self.prompt_singularity_cachedir_remote() else: log.info("Proceeding without consideration of the remote $NXF_SINGULARITY_CACHE index.") self.singularity_cache_index = None @@ -731,7 +731,7 @@ def find_container_images(self, workflow_directory): r"(?<=container)[^\${}]+\${([^{}]+)}(?![^{]*})", contents ) - if bool(container_definition) & bool(container_definition.group(1)): + if bool(container_definition) and bool(container_definition.group(1)): pattern = re.escape(container_definition.group(1)) # extract the quoted string(s) following the variable assignment container_names = re.findall(r"%s\s*=\s*[\"\']([^\"\']+)[\"\']" % pattern, contents) From b634573e7686544409ff837056a20fe421153068 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 5 Jun 2023 11:39:40 +0200 Subject: [PATCH 102/249] validate pipeline parameters with nf-validation --- CHANGELOG.md | 1 + nf_core/lint/files_exist.py | 2 - nf_core/lint/files_unchanged.py | 2 - nf_core/lint/nextflow_config.py | 8 +- .../pipeline-template/lib/NfcoreSchema.groovy | 533 ------------------ .../pipeline-template/lib/WorkflowMain.groovy | 41 -- nf_core/pipeline-template/nextflow.config | 15 +- .../pipeline-template/nextflow_schema.json | 16 +- .../pipeline-template/workflows/pipeline.nf | 23 +- nf_core/schema.py | 8 +- 10 files changed, 55 insertions(+), 594 deletions(-) delete mode 100755 nf_core/pipeline-template/lib/NfcoreSchema.groovy diff --git a/CHANGELOG.md b/CHANGELOG.md index 2deeef5f66..02ef22fcf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) +- Remove schema validation from `lib` folder and use Nextflow nf-validation plugin instead ([#1771](https://github.com/nf-core/tools/pull/1771/)) ### Download diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index eb8c04916a..02baae7db8 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -52,7 +52,6 @@ def files_exist(self): docs/README.md docs/usage.md lib/nfcore_external_java_deps.jar - lib/NfcoreSchema.groovy lib/NfcoreTemplate.groovy lib/Utils.groovy lib/WorkflowMain.groovy @@ -161,7 +160,6 @@ def files_exist(self): [os.path.join("docs", "README.md")], [os.path.join("docs", "usage.md")], [os.path.join("lib", "nfcore_external_java_deps.jar")], - [os.path.join("lib", "NfcoreSchema.groovy")], [os.path.join("lib", "NfcoreTemplate.groovy")], [os.path.join("lib", "Utils.groovy")], [os.path.join("lib", "WorkflowMain.groovy")], diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index c0be64d0d7..2b64d62638 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -40,7 +40,6 @@ def files_unchanged(self): docs/images/nf-core-PIPELINE_logo_dark.png docs/README.md' lib/nfcore_external_java_deps.jar - lib/NfcoreSchema.groovy lib/NfcoreTemplate.groovy ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling @@ -105,7 +104,6 @@ def files_unchanged(self): [os.path.join("docs", "images", f"nf-core-{short_name}_logo_dark.png")], [os.path.join("docs", "README.md")], [os.path.join("lib", "nfcore_external_java_deps.jar")], - [os.path.join("lib", "NfcoreSchema.groovy")], [os.path.join("lib", "NfcoreTemplate.groovy")], ] files_partial = [ diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index af018331f0..d22fa944ed 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -62,11 +62,11 @@ def nextflow_config(self): * Should always be set to default value: ``https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}`` - * ``params.show_hidden_params`` + * ``params.validationShowHiddenParams`` * Determines whether boilerplate params are showed by schema. Set to ``false`` by default - * ``params.schema_ignore_params`` + * ``params.validationSchemaIgnoreParams`` * A comma separated string of inputs the schema validation should ignore. @@ -130,8 +130,8 @@ def nextflow_config(self): ["process.time"], ["params.outdir"], ["params.input"], - ["params.show_hidden_params"], - ["params.schema_ignore_params"], + ["params.validationShowHiddenParams"], + ["params.validationSchemaIgnoreParams"], ] # Throw a warning if these are missing config_warn = [ diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy deleted file mode 100755 index 707f58a276..0000000000 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ /dev/null @@ -1,533 +0,0 @@ -// -// This file holds several functions used to perform JSON parameter validation, help and summary rendering for the nf-core pipeline template. -// - -import nextflow.Nextflow -import org.everit.json.schema.Schema -import org.everit.json.schema.loader.SchemaLoader -import org.everit.json.schema.ValidationException -import org.json.JSONObject -import org.json.JSONTokener -import org.json.JSONArray -import groovy.json.JsonSlurper -import groovy.json.JsonBuilder - -class NfcoreSchema { - - // - // Resolve Schema path relative to main workflow directory - // - public static String getSchemaPath(workflow, schema_filename='nextflow_schema.json') { - return "${workflow.projectDir}/${schema_filename}" - } - - // - // Function to loop over all parameters defined in schema and check - // whether the given parameters adhere to the specifications - // - /* groovylint-disable-next-line UnusedPrivateMethodParameter */ - public static void validateParameters(workflow, params, log, schema_filename='nextflow_schema.json') { - def has_error = false - //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// - // Check for nextflow core params and unexpected params - def json = new File(getSchemaPath(workflow, schema_filename=schema_filename)).text - def Map schemaParams = (Map) new JsonSlurper().parseText(json).get('definitions') - def nf_params = [ - // Options for base `nextflow` command - 'bg', - 'c', - 'C', - 'config', - 'd', - 'D', - 'dockerize', - 'h', - 'log', - 'q', - 'quiet', - 'syslog', - 'v', - - // Options for `nextflow run` command - 'ansi', - 'ansi-log', - 'bg', - 'bucket-dir', - 'c', - 'cache', - 'config', - 'dsl2', - 'dump-channels', - 'dump-hashes', - 'E', - 'entry', - 'latest', - 'lib', - 'main-script', - 'N', - 'name', - 'offline', - 'params-file', - 'pi', - 'plugins', - 'poll-interval', - 'pool-size', - 'profile', - 'ps', - 'qs', - 'queue-size', - 'r', - 'resume', - 'revision', - 'stdin', - 'stub', - 'stub-run', - 'test', - 'w', - 'with-apptainer', - 'with-charliecloud', - 'with-conda', - 'with-dag', - 'with-docker', - 'with-mpi', - 'with-notification', - 'with-podman', - 'with-report', - 'with-singularity', - 'with-timeline', - 'with-tower', - 'with-trace', - 'with-weblog', - 'without-docker', - 'without-podman', - 'work-dir' - ] - def unexpectedParams = [] - - // Collect expected parameters from the schema - def expectedParams = [] - def enums = [:] - for (group in schemaParams) { - for (p in group.value['properties']) { - expectedParams.push(p.key) - if (group.value['properties'][p.key].containsKey('enum')) { - enums[p.key] = group.value['properties'][p.key]['enum'] - } - } - } - - for (specifiedParam in params.keySet()) { - // nextflow params - if (nf_params.contains(specifiedParam)) { - log.error "ERROR: You used a core Nextflow option with two hyphens: '--${specifiedParam}'. Please resubmit with '-${specifiedParam}'" - has_error = true - } - // unexpected params - def params_ignore = params.schema_ignore_params.split(',') + 'schema_ignore_params' - def expectedParamsLowerCase = expectedParams.collect{ it.replace("-", "").toLowerCase() } - def specifiedParamLowerCase = specifiedParam.replace("-", "").toLowerCase() - def isCamelCaseBug = (specifiedParam.contains("-") && !expectedParams.contains(specifiedParam) && expectedParamsLowerCase.contains(specifiedParamLowerCase)) - if (!expectedParams.contains(specifiedParam) && !params_ignore.contains(specifiedParam) && !isCamelCaseBug) { - // Temporarily remove camelCase/camel-case params #1035 - def unexpectedParamsLowerCase = unexpectedParams.collect{ it.replace("-", "").toLowerCase()} - if (!unexpectedParamsLowerCase.contains(specifiedParamLowerCase)){ - unexpectedParams.push(specifiedParam) - } - } - } - - //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// - // Validate parameters against the schema - InputStream input_stream = new File(getSchemaPath(workflow, schema_filename=schema_filename)).newInputStream() - JSONObject raw_schema = new JSONObject(new JSONTokener(input_stream)) - - // Remove anything that's in params.schema_ignore_params - raw_schema = removeIgnoredParams(raw_schema, params) - - Schema schema = SchemaLoader.load(raw_schema) - - // Clean the parameters - def cleanedParams = cleanParameters(params) - - // Convert to JSONObject - def jsonParams = new JsonBuilder(cleanedParams) - JSONObject params_json = new JSONObject(jsonParams.toString()) - - // Validate - try { - schema.validate(params_json) - } catch (ValidationException e) { - println '' - log.error 'ERROR: Validation of pipeline parameters failed!' - JSONObject exceptionJSON = e.toJSON() - printExceptions(exceptionJSON, params_json, log, enums) - println '' - has_error = true - } - - // Check for unexpected parameters - if (unexpectedParams.size() > 0) { - Map colors = NfcoreTemplate.logColours(params.monochrome_logs) - println '' - def warn_msg = 'Found unexpected parameters:' - for (unexpectedParam in unexpectedParams) { - warn_msg = warn_msg + "\n* --${unexpectedParam}: ${params[unexpectedParam].toString()}" - } - log.warn warn_msg - log.info "- ${colors.dim}Ignore this warning: params.schema_ignore_params = \"${unexpectedParams.join(',')}\" ${colors.reset}" - println '' - } - - if (has_error) { - Nextflow.error('Exiting!') - } - } - - // - // Beautify parameters for --help - // - public static String paramsHelp(workflow, params, command, schema_filename='nextflow_schema.json') { - Map colors = NfcoreTemplate.logColours(params.monochrome_logs) - Integer num_hidden = 0 - String output = '' - output += 'Typical pipeline command:\n\n' - output += " ${colors.cyan}${command}${colors.reset}\n\n" - Map params_map = paramsLoad(getSchemaPath(workflow, schema_filename=schema_filename)) - Integer max_chars = paramsMaxChars(params_map) + 1 - Integer desc_indent = max_chars + 14 - Integer dec_linewidth = 160 - desc_indent - for (group in params_map.keySet()) { - Integer num_params = 0 - String group_output = colors.underlined + colors.bold + group + colors.reset + '\n' - def group_params = params_map.get(group) // This gets the parameters of that particular group - for (param in group_params.keySet()) { - if (group_params.get(param).hidden && !params.show_hidden_params) { - num_hidden += 1 - continue; - } - def type = '[' + group_params.get(param).type + ']' - def description = group_params.get(param).description - def defaultValue = group_params.get(param).default != null ? " [default: " + group_params.get(param).default.toString() + "]" : '' - def description_default = description + colors.dim + defaultValue + colors.reset - // Wrap long description texts - // Loosely based on https://dzone.com/articles/groovy-plain-text-word-wrap - if (description_default.length() > dec_linewidth){ - List olines = [] - String oline = "" // " " * indent - description_default.split(" ").each() { wrd -> - if ((oline.size() + wrd.size()) <= dec_linewidth) { - oline += wrd + " " - } else { - olines += oline - oline = wrd + " " - } - } - olines += oline - description_default = olines.join("\n" + " " * desc_indent) - } - group_output += " --" + param.padRight(max_chars) + colors.dim + type.padRight(10) + colors.reset + description_default + '\n' - num_params += 1 - } - group_output += '\n' - if (num_params > 0){ - output += group_output - } - } - if (num_hidden > 0){ - output += colors.dim + "!! Hiding $num_hidden params, use --show_hidden_params to show them !!\n" + colors.reset - } - output += NfcoreTemplate.dashedLine(params.monochrome_logs) - return output - } - - // - // Groovy Map summarising parameters/workflow options used by the pipeline - // - public static LinkedHashMap paramsSummaryMap(workflow, params, schema_filename='nextflow_schema.json') { - // Get a selection of core Nextflow workflow options - def Map workflow_summary = [:] - if (workflow.revision) { - workflow_summary['revision'] = workflow.revision - } - workflow_summary['runName'] = workflow.runName - if (workflow.containerEngine) { - workflow_summary['containerEngine'] = workflow.containerEngine - } - if (workflow.container) { - workflow_summary['container'] = workflow.container - } - workflow_summary['launchDir'] = workflow.launchDir - workflow_summary['workDir'] = workflow.workDir - workflow_summary['projectDir'] = workflow.projectDir - workflow_summary['userName'] = workflow.userName - workflow_summary['profile'] = workflow.profile - workflow_summary['configFiles'] = workflow.configFiles.join(', ') - - // Get pipeline parameters defined in JSON Schema - def Map params_summary = [:] - def params_map = paramsLoad(getSchemaPath(workflow, schema_filename=schema_filename)) - for (group in params_map.keySet()) { - def sub_params = new LinkedHashMap() - def group_params = params_map.get(group) // This gets the parameters of that particular group - for (param in group_params.keySet()) { - if (params.containsKey(param)) { - def params_value = params.get(param) - def schema_value = group_params.get(param).default - def param_type = group_params.get(param).type - if (schema_value != null) { - if (param_type == 'string') { - if (schema_value.contains('$projectDir') || schema_value.contains('${projectDir}')) { - def sub_string = schema_value.replace('\$projectDir', '') - sub_string = sub_string.replace('\${projectDir}', '') - if (params_value.contains(sub_string)) { - schema_value = params_value - } - } - if (schema_value.contains('$params.outdir') || schema_value.contains('${params.outdir}')) { - def sub_string = schema_value.replace('\$params.outdir', '') - sub_string = sub_string.replace('\${params.outdir}', '') - if ("${params.outdir}${sub_string}" == params_value) { - schema_value = params_value - } - } - } - } - - // We have a default in the schema, and this isn't it - if (schema_value != null && params_value != schema_value) { - sub_params.put(param, params_value) - } - // No default in the schema, and this isn't empty - else if (schema_value == null && params_value != "" && params_value != null && params_value != false) { - sub_params.put(param, params_value) - } - } - } - params_summary.put(group, sub_params) - } - return [ 'Core Nextflow options' : workflow_summary ] << params_summary - } - - // - // Beautify parameters for summary and return as string - // - public static String paramsSummaryLog(workflow, params) { - Map colors = NfcoreTemplate.logColours(params.monochrome_logs) - String output = '' - def params_map = paramsSummaryMap(workflow, params) - def max_chars = paramsMaxChars(params_map) - for (group in params_map.keySet()) { - def group_params = params_map.get(group) // This gets the parameters of that particular group - if (group_params) { - output += colors.bold + group + colors.reset + '\n' - for (param in group_params.keySet()) { - output += " " + colors.blue + param.padRight(max_chars) + ": " + colors.green + group_params.get(param) + colors.reset + '\n' - } - output += '\n' - } - } - output += "!! Only displaying parameters that differ from the pipeline defaults !!\n" - {% if igenomes -%} - output += "Displaying all reference genome parameters\n" - {% endif -%} - output += NfcoreTemplate.dashedLine(params.monochrome_logs) - return output - } - - // - // Loop over nested exceptions and print the causingException - // - private static void printExceptions(ex_json, params_json, log, enums, limit=5) { - def causingExceptions = ex_json['causingExceptions'] - if (causingExceptions.length() == 0) { - def m = ex_json['message'] =~ /required key \[([^\]]+)\] not found/ - // Missing required param - if (m.matches()) { - log.error "* Missing required parameter: --${m[0][1]}" - } - // Other base-level error - else if (ex_json['pointerToViolation'] == '#') { - log.error "* ${ex_json['message']}" - } - // Error with specific param - else { - def param = ex_json['pointerToViolation'] - ~/^#\// - def param_val = params_json[param].toString() - if (enums.containsKey(param)) { - def error_msg = "* --${param}: '${param_val}' is not a valid choice (Available choices" - if (enums[param].size() > limit) { - log.error "${error_msg} (${limit} of ${enums[param].size()}): ${enums[param][0..limit-1].join(', ')}, ... )" - } else { - log.error "${error_msg}: ${enums[param].join(', ')})" - } - } else { - log.error "* --${param}: ${ex_json['message']} (${param_val})" - } - } - } - for (ex in causingExceptions) { - printExceptions(ex, params_json, log, enums) - } - } - - // - // Remove an element from a JSONArray - // - private static JSONArray removeElement(json_array, element) { - def list = [] - int len = json_array.length() - for (int i=0;i - if(raw_schema.keySet().contains('definitions')){ - raw_schema.definitions.each { definition -> - for (key in definition.keySet()){ - if (definition[key].get("properties").keySet().contains(ignore_param)){ - // Remove the param to ignore - definition[key].get("properties").remove(ignore_param) - // If the param was required, change this - if (definition[key].has("required")) { - def cleaned_required = removeElement(definition[key].required, ignore_param) - definition[key].put("required", cleaned_required) - } - } - } - } - } - if(raw_schema.keySet().contains('properties') && raw_schema.get('properties').keySet().contains(ignore_param)) { - raw_schema.get("properties").remove(ignore_param) - } - if(raw_schema.keySet().contains('required') && raw_schema.required.contains(ignore_param)) { - def cleaned_required = removeElement(raw_schema.required, ignore_param) - raw_schema.put("required", cleaned_required) - } - } - return raw_schema - } - - // - // Clean and check parameters relative to Nextflow native classes - // - private static Map cleanParameters(params) { - def new_params = params.getClass().newInstance(params) - for (p in params) { - // remove anything evaluating to false - if (!p['value']) { - new_params.remove(p.key) - } - // Cast MemoryUnit to String - if (p['value'].getClass() == nextflow.util.MemoryUnit) { - new_params.replace(p.key, p['value'].toString()) - } - // Cast Duration to String - if (p['value'].getClass() == nextflow.util.Duration) { - new_params.replace(p.key, p['value'].toString().replaceFirst(/d(?!\S)/, "day")) - } - // Cast LinkedHashMap to String - if (p['value'].getClass() == LinkedHashMap) { - new_params.replace(p.key, p['value'].toString()) - } - } - return new_params - } - - // - // This function tries to read a JSON params file - // - private static LinkedHashMap paramsLoad(String json_schema) { - def params_map = new LinkedHashMap() - try { - params_map = paramsRead(json_schema) - } catch (Exception e) { - println "Could not read parameters settings from JSON. $e" - params_map = new LinkedHashMap() - } - return params_map - } - - // - // Method to actually read in JSON file using Groovy. - // Group (as Key), values are all parameters - // - Parameter1 as Key, Description as Value - // - Parameter2 as Key, Description as Value - // .... - // Group - // - - private static LinkedHashMap paramsRead(String json_schema) throws Exception { - def json = new File(json_schema).text - def Map schema_definitions = (Map) new JsonSlurper().parseText(json).get('definitions') - def Map schema_properties = (Map) new JsonSlurper().parseText(json).get('properties') - /* Tree looks like this in nf-core schema - * definitions <- this is what the first get('definitions') gets us - group 1 - title - description - properties - parameter 1 - type - description - parameter 2 - type - description - group 2 - title - description - properties - parameter 1 - type - description - * properties <- parameters can also be ungrouped, outside of definitions - parameter 1 - type - description - */ - - // Grouped params - def params_map = new LinkedHashMap() - schema_definitions.each { key, val -> - def Map group = schema_definitions."$key".properties // Gets the property object of the group - def title = schema_definitions."$key".title - def sub_params = new LinkedHashMap() - group.each { innerkey, value -> - sub_params.put(innerkey, value) - } - params_map.put(title, sub_params) - } - - // Ungrouped params - def ungrouped_params = new LinkedHashMap() - schema_properties.each { innerkey, value -> - ungrouped_params.put(innerkey, value) - } - params_map.put("Other parameters", ungrouped_params) - - return params_map - } - - // - // Get maximum number of characters across all parameter names - // - private static Integer paramsMaxChars(params_map) { - Integer max_chars = 0 - for (group in params_map.keySet()) { - def group_params = params_map.get(group) // This gets the parameters of that particular group - for (param in group_params.keySet()) { - if (param.size() > max_chars) { - max_chars = param.size() - } - } - } - return max_chars - } -} diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 4cb7409fb9..3136cdf16b 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -20,45 +20,11 @@ class WorkflowMain { " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" } - // - // Generate help string - // - public static String help(workflow, params) { - {% if igenomes -%} - def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" - {% else -%} - def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --fasta reference.fa -profile docker" - {% endif -%} - def help_string = '' - help_string += NfcoreTemplate.logo(workflow, params.monochrome_logs) - help_string += NfcoreSchema.paramsHelp(workflow, params, command) - help_string += '\n' + citation(workflow) + '\n' - help_string += NfcoreTemplate.dashedLine(params.monochrome_logs) - return help_string - } - - // - // Generate parameter summary log string - // - public static String paramsSummaryLog(workflow, params) { - def summary_log = '' - summary_log += NfcoreTemplate.logo(workflow, params.monochrome_logs) - summary_log += NfcoreSchema.paramsSummaryLog(workflow, params) - summary_log += '\n' + citation(workflow) + '\n' - summary_log += NfcoreTemplate.dashedLine(params.monochrome_logs) - return summary_log - } // // Validate parameters and print summary to screen // public static void initialise(workflow, params, log) { - // Print help to screen if required - if (params.help) { - log.info help(workflow, params) - System.exit(0) - } - // Print workflow version and exit on --version if (params.version) { String workflow_version = NfcoreTemplate.version(workflow) @@ -66,13 +32,6 @@ class WorkflowMain { System.exit(0) } - // Print parameter summary log to screen - log.info paramsSummaryLog(workflow, params) - - // Validate workflow parameters via the JSON schema - if (params.validate_params) { - NfcoreSchema.validateParameters(workflow, params, log) - } // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2650572c93..28dcc80609 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -38,9 +38,6 @@ params { hook_url = null help = false version = false - validate_params = true - show_hidden_params = false - schema_ignore_params = 'genomes' {% if nf_core_configs %} // Config options @@ -58,6 +55,13 @@ params { max_cpus = 16 max_time = '240.h' + // Schema validation default options + validationFailUnrecognisedParams = false + validationLenientMode = false + validationSchemaIgnoreParams = 'genomes' + validationShowHiddenParams = false + validate_params = true + } // Load base.config by default for all pipelines @@ -179,6 +183,11 @@ profiles { docker.registry = 'quay.io' podman.registry = 'quay.io' +// Nextflow plugins +plugins { + id 'nf-validation@0.2.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet +} + {% if igenomes %} // Load igenomes.config if required if (!params.igenomes_ignore) { diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 2743562d6c..d3ad943a5d 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -257,12 +257,26 @@ "fa_icon": "fas fa-check-square", "hidden": true }, - "show_hidden_params": { + "validationShowHiddenParams": { "type": "boolean", "fa_icon": "far fa-eye-slash", "description": "Show all params when using `--help`", "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." + }, + "validationFailUnrecognisedParams": { + "type": "boolean", + "fa_icon": "far fa-check-circle", + "description": "Validation of parameters fails when an unrecognised parameter is found.", + "hidden": true, + "help_text": "By default, when an unrecognised parameter is found, it returns a warinig." + }, + "validationLenientMode": { + "type": "boolean", + "fa_icon": "far fa-check-circle", + "description": "Validation of parameters in lenient more.", + "hidden": true, + "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." } } } diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 9bcc0086b5..e36dd763e5 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -4,9 +4,27 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -def summary_params = NfcoreSchema.paramsSummaryMap(workflow, params) +include { validateParameters; paramsHelp; paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' + +def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) +def citation = '\n' + WorkflowMain.citation(workflow) + '\n' +def summary_params = paramsSummaryMap(workflow) + +// Print help message if needed +if (params.help) { + def String command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" + log.info logo + paramsHelp(command) + citation + NfcoreTemplate.dashedLine(params.monochrome_logs) + System.exit(0) +} // Validate input parameters +if (params.validate_params) { + validateParameters() +} + +// Print parameter summary log to screen +log.info logo + paramsSummaryLog(workflow) + citation + Workflow{{ short_name[0]|upper }}{{ short_name[1:] }}.initialise(params, log) // TODO nf-core: Add all file path parameters for the pipeline to the list below @@ -14,9 +32,6 @@ Workflow{{ short_name[0]|upper }}{{ short_name[1:] }}.initialise(params, log) def checkPathParamList = [ params.input, params.multiqc_config, params.fasta ] for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true) } } -// Check mandatory parameters -if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONFIG FILES diff --git a/nf_core/schema.py b/nf_core/schema.py index ba88e762ea..75dbebce04 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -245,8 +245,8 @@ def validate_default_params(self): self.get_wf_params() # Collect parameters to ignore - if "schema_ignore_params" in self.pipeline_params: - params_ignore = self.pipeline_params.get("schema_ignore_params", "").strip("\"'").split(",") + if "validationSchemaIgnoreParams" in self.pipeline_params: + params_ignore = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") else: params_ignore = [] @@ -759,8 +759,8 @@ def add_schema_found_configs(self): Add anything that's found in the Nextflow params that's missing in the pipeline schema """ params_added = [] - params_ignore = self.pipeline_params.get("schema_ignore_params", "").strip("\"'").split(",") - params_ignore.append("schema_ignore_params") + params_ignore = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") + params_ignore.append("validationSchemaIgnoreParams") for p_key, p_val in self.pipeline_params.items(): # Check if key is in schema parameters if p_key not in self.schema_params and p_key not in params_ignore: From df791970d12501cc595ead58085153bf35840f76 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 5 Jun 2023 11:57:03 +0200 Subject: [PATCH 103/249] fix ch_input --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index e36dd763e5..ba4baf3a53 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -84,7 +84,7 @@ workflow {{ short_name|upper }} { // SUBWORKFLOW: Read in samplesheet, validate and stage input files // INPUT_CHECK ( - ch_input + file(params.input) ) ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) From 6e4c582c2d502ca869d2be29cde3e827a72daf1a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 5 Jun 2023 11:57:41 +0200 Subject: [PATCH 104/249] add parameter defaults to schema to avoid printing them with parameter summary --- nf_core/pipeline-template/nextflow_schema.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index d3ad943a5d..d7db5abea3 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -174,12 +174,14 @@ "type": "boolean", "description": "Display help text.", "fa_icon": "fas fa-question-circle", + "default": false, "hidden": true }, "version": { "type": "boolean", "description": "Display version and exit.", "fa_icon": "fas fa-question-circle", + "default": false, "hidden": true }, "publish_dir_mode": { @@ -203,6 +205,7 @@ "type": "boolean", "description": "Send plain-text email instead of HTML.", "fa_icon": "fas fa-remove-format", + "default": false, "hidden": true }, "max_multiqc_email_size": { @@ -217,6 +220,7 @@ "type": "boolean", "description": "Do not use coloured log outputs.", "fa_icon": "fas fa-palette", + "default": false, "hidden": true }, "hook_url": { @@ -261,6 +265,7 @@ "type": "boolean", "fa_icon": "far fa-eye-slash", "description": "Show all params when using `--help`", + "default": false, "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." }, @@ -268,6 +273,7 @@ "type": "boolean", "fa_icon": "far fa-check-circle", "description": "Validation of parameters fails when an unrecognised parameter is found.", + "default": false, "hidden": true, "help_text": "By default, when an unrecognised parameter is found, it returns a warinig." }, @@ -275,6 +281,7 @@ "type": "boolean", "fa_icon": "far fa-check-circle", "description": "Validation of parameters in lenient more.", + "default": false, "hidden": true, "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." } From 2cf08142220f27dd2e1445acb78d1c5627e5a09c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 5 Jun 2023 13:20:56 +0200 Subject: [PATCH 105/249] check if input file exists --- nf_core/pipeline-template/nextflow_schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index d7db5abea3..fd29e64cda 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -15,6 +15,7 @@ "input": { "type": "string", "format": "file-path", + "exists": true, "mimetype": "text/csv", "pattern": "^\\S+\\.csv$", "schema": "assets/schema_input.json", From 83533409da603c1dca959296c63bbd510ff48b8a Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 6 Jun 2023 10:21:30 +0000 Subject: [PATCH 106/249] Update seqeralabs/action-tower-launch to v1.2.4 - Updates the Github action to v1.2.4 - Adds specific revision which no longer defaults to github.sha - Saves the JSON log created per pipeline as artifact --- CHANGELOG.md | 1 + .../pipeline-template/.github/workflows/awsfulltest.yml | 8 ++++++-- nf_core/pipeline-template/.github/workflows/awstest.yml | 8 ++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2deeef5f66..a0584f3e3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) +- Updates seqeralabs/action-tower-launch to v1.2.4 ### Download diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 9511ea57be..de2fb535a5 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1 + uses: seqeralabs/action-tower-launch@v1.2.4 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters {%- raw %} @@ -22,13 +22,17 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} parameters: | { "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" } profiles: test_full + - uses: actions/upload-artifact@v3 with: name: Tower debug log file - path: tower_action_*.log + path: | + tower_action_*.log + tower_action_*.json diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 6e1fd8c92a..6e5a9a9e42 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -12,18 +12,22 @@ jobs: steps: # Launch workflow using Tower CLI tool action {%- raw %} - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1 + uses: seqeralabs/action-tower-launch@v1.2.4 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} parameters: | { "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-test-${{ github.sha }}{% endraw %}" } profiles: test + - uses: actions/upload-artifact@v3 with: name: Tower debug log file - path: tower_action_*.log + path: | + tower_action_*.log + tower_action_*.json From 410db78bf8fdddb8302256a5c08a5bbb5da60fe6 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 6 Jun 2023 15:33:47 +0000 Subject: [PATCH 107/249] Update action to v2.0.0 (breaking change) --- CHANGELOG.md | 2 +- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 2 +- nf_core/pipeline-template/.github/workflows/awstest.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0584f3e3a..8a1fb5ae28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) -- Updates seqeralabs/action-tower-launch to v1.2.4 +- Updates seqeralabs/action-tower-launch to v2.0.0 ([#2301](https://github.com/nf-core/tools/pull/2301)) ### Download diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index de2fb535a5..ebad007191 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1.2.4 + uses: seqeralabs/action-tower-launch@v2.0.0 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters {%- raw %} diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 6e5a9a9e42..8819b7cca0 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -12,7 +12,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action {%- raw %} - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1.2.4 + uses: seqeralabs/action-tower-launch@v2.0.0 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} From 05193076f05c095b87d242bd5ffacc9dc0ed468b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 7 Jun 2023 09:07:28 +0200 Subject: [PATCH 108/249] update nf-validation to v0.2.1 --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 28dcc80609..f8aa90b4e7 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -185,7 +185,7 @@ podman.registry = 'quay.io' // Nextflow plugins plugins { - id 'nf-validation@0.2.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-validation@0.2.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet } {% if igenomes %} From 81595ecd3d4c1ccdf336914eaf73626fa3da6f8b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 7 Jun 2023 09:10:06 +0200 Subject: [PATCH 109/249] check paths exist with schema --- nf_core/pipeline-template/nextflow_schema.json | 2 ++ nf_core/pipeline-template/workflows/pipeline.nf | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index fd29e64cda..1b5063bb56 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -58,6 +58,7 @@ "fasta": { "type": "string", "format": "file-path", + "exists": true, "mimetype": "text/plain", "pattern": "^\\S+\\.fn?a(sta)?(\\.gz)?$", "description": "Path to FASTA genome file.", @@ -235,6 +236,7 @@ "type": "string", "description": "Custom config file to supply to MultiQC.", "fa_icon": "fas fa-cog", + "exists": true, "hidden": true }, "multiqc_logo": { diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index ba4baf3a53..dd90e003c1 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -27,11 +27,6 @@ log.info logo + paramsSummaryLog(workflow) + citation Workflow{{ short_name[0]|upper }}{{ short_name[1:] }}.initialise(params, log) -// TODO nf-core: Add all file path parameters for the pipeline to the list below -// Check input path parameters to see if they exist -def checkPathParamList = [ params.input, params.multiqc_config, params.fasta ] -for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true) } } - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONFIG FILES From e749f08610e324dc03fcbc5b211f6e8f7a9061a9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 7 Jun 2023 09:15:53 +0200 Subject: [PATCH 110/249] add comment about creating a channel with fromSamplesheet --- nf_core/pipeline-template/workflows/pipeline.nf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index dd90e003c1..ca78758fc8 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -82,6 +82,9 @@ workflow {{ short_name|upper }} { file(params.input) ) ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) + // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the sample sheet with Channel.fromSamplesheet("input") + // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ + // ! There is currently no tooling to help you write a sample sheet schema // // MODULE: Run FastQC From d9b7b27acd920ba8beafe73d5b4dc50e65450b13 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 7 Jun 2023 11:07:10 +0200 Subject: [PATCH 111/249] print help before initialise --- .../pipeline-template/lib/WorkflowMain.groovy | 6 +++++- .../lib/WorkflowPipeline.groovy | 2 +- nf_core/pipeline-template/main.nf | 16 ++++++++++++++++ .../pipeline-template/workflows/pipeline.nf | 18 +++--------------- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 3136cdf16b..d6e78fb11f 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -25,6 +25,11 @@ class WorkflowMain { // Validate parameters and print summary to screen // public static void initialise(workflow, params, log) { + // Print help to screen if required + if (params.help) { + return + } + // Print workflow version and exit on --version if (params.version) { String workflow_version = NfcoreTemplate.version(workflow) @@ -32,7 +37,6 @@ class WorkflowMain { System.exit(0) } - // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index 6e577a669c..2cf5e68fdd 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -13,7 +13,7 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { public static void initialise(params, log) { {% if igenomes -%} genomeExistsError(params, log) -{% endif %} + {% endif %} if (!params.fasta) { Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 52d8b1bb38..561f2f6384 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -27,6 +27,22 @@ params.fasta = WorkflowMain.getGenomeAttribute(params, 'fasta') ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +include { validateParameters; paramsHelp } from 'plugin/nf-validation' + +// Print help message if needed +if (params.help) { + def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) + def citation = '\n' + WorkflowMain.citation(workflow) + '\n' + def String command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" + log.info logo + paramsHelp(command) + citation + NfcoreTemplate.dashedLine(params.monochrome_logs) + System.exit(0) +} + +// Validate input parameters +if (params.validate_params) { + validateParameters() +} + WorkflowMain.initialise(workflow, params, log) /* diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index ca78758fc8..72d9825a1c 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -1,27 +1,15 @@ /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - VALIDATE INPUTS + PRINT PARAMS SUMMARY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { validateParameters; paramsHelp; paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' +include { paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) def citation = '\n' + WorkflowMain.citation(workflow) + '\n' def summary_params = paramsSummaryMap(workflow) -// Print help message if needed -if (params.help) { - def String command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" - log.info logo + paramsHelp(command) + citation + NfcoreTemplate.dashedLine(params.monochrome_logs) - System.exit(0) -} - -// Validate input parameters -if (params.validate_params) { - validateParameters() -} - // Print parameter summary log to screen log.info logo + paramsSummaryLog(workflow) + citation @@ -82,7 +70,7 @@ workflow {{ short_name|upper }} { file(params.input) ) ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) - // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the sample sheet with Channel.fromSamplesheet("input") + // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the samplesheet with Channel.fromSamplesheet("input") // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ // ! There is currently no tooling to help you write a sample sheet schema From e36fdbfef3d739e30ac4d8ac244f3edce29834da Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 7 Jun 2023 11:14:39 +0200 Subject: [PATCH 112/249] fix trailing whitespaces --- nf_core/pipeline-template/lib/WorkflowPipeline.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index 2cf5e68fdd..6e577a669c 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -13,7 +13,7 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { public static void initialise(params, log) { {% if igenomes -%} genomeExistsError(params, log) - {% endif %} +{% endif %} if (!params.fasta) { Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." From e74f588fe41555f90c8c63c18ed3832b24337f6f Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 8 Jun 2023 09:00:50 +0200 Subject: [PATCH 113/249] Adds `d` to the pattern of max_time --- nf_core/pipeline-template/nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 2743562d6c..be401a4fed 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -157,7 +157,7 @@ "description": "Maximum amount of time that can be requested for any single job.", "default": "240.h", "fa_icon": "far fa-clock", - "pattern": "^(\\d+\\.?\\s*(s|m|h|day)\\s*)+$", + "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$", "hidden": true, "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`" } From e40580ba71445049caa786fb5b7d4dd639c8713b Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Thu, 8 Jun 2023 08:59:02 +0000 Subject: [PATCH 114/249] Reduce to shortened version of action-tower-launch version --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 2 +- nf_core/pipeline-template/.github/workflows/awstest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index ebad007191..1d5c53bda9 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v2.0.0 + uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters {%- raw %} diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 8819b7cca0..9a0bf4afbc 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -12,7 +12,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action {%- raw %} - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v2.0.0 + uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} From d5b7bd220e96d030c891a307cbc30117a7bc1eca Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Jun 2023 11:19:32 +0100 Subject: [PATCH 115/249] Add singularity.registry = 'quay.io' to pipeline template --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- CHANGELOG.md | 2 ++ nf_core/lint/readme.py | 2 +- .../.github/ISSUE_TEMPLATE/bug_report.yml | 2 +- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- nf_core/pipeline-template/README.md | 2 +- nf_core/pipeline-template/nextflow.config | 7 ++++--- 9 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 1bd2e27fb0..8fdd2bd7e1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -33,7 +33,7 @@ body: attributes: label: System information description: | - * Nextflow version _(eg. 22.10.1)_ + * Nextflow version _(eg. 23.04.0)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 46152d92a7..654d248f79 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" + - "23.04.0" - "latest-everything" steps: # Get the repo code diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 8f3c5fdb47..7cff154a08 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" + - "23.04.0" - "latest-everything" steps: - uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2deeef5f66..9572c95440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) +- Add `singularity.registry = 'quay.io'` in pipeline template +- Bump minimum required NF version in pipeline template from `22.10.1` -> `23.04.0` ### Download diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py index ae5c542837..55060442b1 100644 --- a/nf_core/lint/readme.py +++ b/nf_core/lint/readme.py @@ -36,7 +36,7 @@ def readme(self): if "nextflow_badge" not in ignore_configs: # Check that there is a readme badge showing the minimum required version of Nextflow - # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/) + # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) # and that it has the correct version nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" match = re.search(nf_badge_re, content) diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml index f93cd55d59..063690f29f 100644 --- a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml @@ -42,7 +42,7 @@ body: attributes: label: System information description: | - * Nextflow version _(eg. 22.10.1)_ + * Nextflow version _(eg. 23.04.0)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_ diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index bf3dc36bc5..521f3e664a 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" + - "23.04.0" - "latest-everything" steps: - name: Check out pipeline code diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 5ac65123ed..e66746dcd5 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -10,7 +10,7 @@ {%- if github_badges -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2650572c93..37137bb027 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -176,8 +176,9 @@ profiles { // Set default registry for Docker and Podman independent of -profile // Will not be used unless Docker / Podman are enabled // Set to your registry if you have a mirror of containers -docker.registry = 'quay.io' -podman.registry = 'quay.io' +singularity.registry = 'quay.io' +docker.registry = 'quay.io' +podman.registry = 'quay.io' {% if igenomes %} // Load igenomes.config if required @@ -226,7 +227,7 @@ manifest { homePage = 'https://github.com/{{ name }}' description = """{{ description }}""" mainScript = 'main.nf' - nextflowVersion = '!>=22.10.1' + nextflowVersion = '!>=23.04.0' version = '{{ version }}' doi = '' } From 114308419617c38654599d062d4befa2780f8840 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 13:09:46 +0100 Subject: [PATCH 116/249] fix: Parsing of container directive Changes: - Strip additional components off container directive prior to parsing - Helps parse additional strings that may be present at the front of container directive (e.g. 'container '). - Should catch more non-standard container directives (of which there are many). --- CHANGELOG.md | 1 + nf_core/modules/lint/main_nf.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2deeef5f66..d99229df93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) +- Fix parsing of container directive when it is not typical nf-core format. ### Download diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 31b8adca3a..cfbc2c4ad1 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -240,7 +240,7 @@ def check_process_section(self, lines, fix_version, progress_bar): # Deprecated enable_conda for i, l in enumerate(lines): url = None - l = l.strip(" '\"") + l = l.strip().removeprefix("container ").strip(" \n'\"") if _container_type(l) == "conda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] match = re.search(r"params\.enable_conda", l) @@ -263,7 +263,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?'", l) + match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) @@ -275,13 +275,13 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 - match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)'", l) + match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)", l) if match is not None: docker_tag = match.group(1) self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) - docker_tag = NoneD + docker_tag = None if l.startswith("quay.io/"): l_stripped = re.sub(r"\W+$", "", l) self.failed.append( From da476f2c76e28371ebe54eb54068369963848f95 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 13:16:11 +0100 Subject: [PATCH 117/249] CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d99229df93..d6599be574 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) -- Fix parsing of container directive when it is not typical nf-core format. +- Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306)) ### Download From c98848a8bb088e1d6c75759fe8444c165f118348 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 13:59:10 +0100 Subject: [PATCH 118/249] Removed 'removeprefix' which is Py > 3.9 only --- nf_core/modules/lint/main_nf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index cfbc2c4ad1..c6ea2ed6c3 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -240,7 +240,12 @@ def check_process_section(self, lines, fix_version, progress_bar): # Deprecated enable_conda for i, l in enumerate(lines): url = None - l = l.strip().removeprefix("container ").strip(" \n'\"") + l = l.strip(" \n'\"") + + # Catch preceeding "container " + if l.startswith("container"): + l = l.replace("container", "").strip(" \n'\"") + if _container_type(l) == "conda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] match = re.search(r"params\.enable_conda", l) From d454093e514d9595bd3a3f0def71fe9eefdc8e28 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 8 Jun 2023 17:30:17 +0200 Subject: [PATCH 119/249] The repo must not be in detached state for Tower's UI. --- nf_core/download.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 70f61f35a4..6fed091948 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -157,7 +157,7 @@ def download_workflow(self): sys.exit(1) summary_log = [ - f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',...,['+str(len(self.revision)-2)+' more revisions],...,'+self.revision[-1]}'", + f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", f"Pull containers: '{self.container}'", ] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: @@ -1268,9 +1268,14 @@ def tidy_tags_and_branches(self): for head in heads_to_remove: self.repo.delete_head(head) - # ensure all desired branches are available + # ensure all desired revisions/branches are available for revision in desired_revisions: - self.checkout(revision) + if self.repo.is_valid_object(revision): + self.repo.create_head(revision, revision) + self.checkout(revision) + if self.repo.head.is_detached: + self.repo.head.reset(index=True, working_tree=True) + self.heads = self.repo.heads # get all tags and available remote_branches From 8fe15ea2301c9d391ffcfd46957b3abb392dfe5c Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 16:52:52 +0100 Subject: [PATCH 120/249] Use specific Nextflow + nf-amazon version to get check if tests pass --- .github/workflows/create-test-wf.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 8f3c5fdb47..d52fa22301 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,8 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" - - "latest-everything" + - "23.04.1" steps: - uses: actions/checkout@v3 name: Check out source-code repository @@ -47,7 +46,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - nextflow run nf-core-testpipeline -profile test,docker --outdir ./results + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results -plugins nf-amazon@1.16.2 - name: Upload log file artifact if: ${{ always() }} From d24652af1f7e9dd657bfe0d68b4b637b0c2125b9 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 16:52:52 +0100 Subject: [PATCH 121/249] Use specific Nextflow + nf-amazon version to get check if tests pass --- .github/workflows/create-test-wf.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 8f3c5fdb47..d52fa22301 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,8 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" - - "latest-everything" + - "23.04.1" steps: - uses: actions/checkout@v3 name: Check out source-code repository @@ -47,7 +46,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - nextflow run nf-core-testpipeline -profile test,docker --outdir ./results + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results -plugins nf-amazon@1.16.2 - name: Upload log file artifact if: ${{ always() }} From 130aac7f6e03185734a1e4d5cabe1d71493d1f38 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 8 Jun 2023 18:53:18 +0200 Subject: [PATCH 122/249] Introduce latest branch for all Tower downloads, bugix in prompt_singularity_cachedir_remote() --- nf_core/download.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index f644d1c188..e5386126ad 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -13,6 +13,7 @@ import tarfile import textwrap from datetime import datetime +from distutils.version import StrictVersion from zipfile import ZipFile import git @@ -531,6 +532,7 @@ def read_remote_containers(self): except (FileNotFoundError, LookupError) as e: log.error(f"[red]Issue with reading the specified remote $NXF_SINGULARITY_CACHE index:[/]\n{e}\n") if stderr.is_interactive and rich.prompt.Confirm.ask(f"[blue]Specify a new index file and try again?"): + self.singularity_cache_index = None # reset chosen path to index file. self.prompt_singularity_cachedir_remote() else: log.info("Proceeding without consideration of the remote $NXF_SINGULARITY_CACHE index.") @@ -1271,10 +1273,14 @@ def tidy_tags_and_branches(self): # ensure all desired revisions/branches are available for revision in desired_revisions: if self.repo.is_valid_object(revision): - self.repo.create_head(revision, revision) self.checkout(revision) - if self.repo.head.is_detached: - self.repo.head.reset(index=True, working_tree=True) + + # create "latest" branch to ensure at least one branch exists (required for Tower's UI to display revisions correctly) + latest = sorted(desired_revisions, key=StrictVersion)[-1] + self.repo.create_head("latest", latest) + self.checkout(latest) + if self.repo.head.is_detached: + self.repo.head.reset(index=True, working_tree=True) self.heads = self.repo.heads From 27a18b909a032847b6ce687104e24efb9ad95fdd Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 8 Jun 2023 19:24:46 +0200 Subject: [PATCH 123/249] Bugfix in with file validation in prompt_singularity_cachedir_remote(). Type error, because it needs to be callable and not a plain string. --- nf_core/download.py | 10 +++++++--- nf_core/utils.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index e5386126ad..ee1e6e523f 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -28,7 +28,11 @@ import nf_core.list import nf_core.utils from nf_core.synced_repo import RemoteProgressbar, SyncedRepo -from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR +from nf_core.utils import ( + NFCORE_CACHE_DIR, + NFCORE_DIR, + SingularityCacheFilePathValidator, +) log = logging.getLogger(__name__) stderr = rich.console.Console( @@ -494,8 +498,8 @@ def prompt_singularity_cachedir_remote(self): cachedir_index = None while cachedir_index is None: prompt_cachedir_index = questionary.path( - "Specify a list of the remote images already present in the remote system :", - file_filter="*.txt", + "Specify a list of the container images that are already present on the remote system:", + validate=SingularityCacheFilePathValidator, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() cachedir_index = os.path.abspath(os.path.expanduser(prompt_cachedir_index)) diff --git a/nf_core/utils.py b/nf_core/utils.py index 3ddce9b870..ba0a4fabf3 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -880,6 +880,23 @@ def prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False): ) +class SingularityCacheFilePathValidator(questionary.Validator): + """ + Validator for file path specified as --singularity-cache-index argument in nf-core download + """ + + def validate(self, value): + if len(value.text): + if os.path.isfile(value.text): + return True + else: + raise questionary.ValidationError( + message="Invalid remote cache index file", cursor_position=len(value.text) + ) + else: + return True + + def get_repo_releases_branches(pipeline, wfs): """Fetches details of a nf-core workflow to download. From 9ce5ce4a7d1b760ac51b9e24c5ef847b94a48adc Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 8 Jun 2023 19:49:14 +0200 Subject: [PATCH 124/249] Adapting the test to the new behavior. --- tests/test_download.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_download.py b/tests/test_download.py index aa2e959f3d..4ee8d3ef6e 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -261,8 +261,14 @@ def test_download_workflow_for_tower(self, tmp_dir): assert download_obj.workflow_repo assert isinstance(download_obj.workflow_repo, WorkflowRepo) assert issubclass(type(download_obj.workflow_repo), SyncedRepo) + # corroborate that the other revisions are inaccessible to the user. - assert len(download_obj.workflow_repo.tags) == len(download_obj.revision) + all_tags = {tag.name for tag in download_obj.workflow_repo.tags} + all_heads = {head.name for head in download_obj.workflow_repo.heads} + + assert set(download_obj.revision) == all_tags + # assert that the download has a "latest" branch. + assert "latest" in all_heads # download_obj.download_workflow_tower(location=tmp_dir) will run container image detection for all requested revisions assert isinstance(download_obj.containers, list) and len(download_obj.containers) == 33 From b4f86d5bd9c052b6edde8bd2f3604dcd1c95a9f0 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Jun 2023 19:55:50 +0100 Subject: [PATCH 125/249] Update create-test-wf.yml --- .github/workflows/create-test-wf.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index d52fa22301..8f3c5fdb47 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,7 +23,8 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.1" + - "22.10.1" + - "latest-everything" steps: - uses: actions/checkout@v3 name: Check out source-code repository @@ -46,7 +47,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - nextflow run nf-core-testpipeline -profile test,docker --outdir ./results -plugins nf-amazon@1.16.2 + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results - name: Upload log file artifact if: ${{ always() }} From 44a7c7dc5c796be5f30da8c13d42ffe9f03d54f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 9 Jun 2023 10:01:26 +0200 Subject: [PATCH 126/249] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- CHANGELOG.md | 2 +- nf_core/pipeline-template/lib/WorkflowMain.groovy | 4 ---- nf_core/pipeline-template/nextflow_schema.json | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02ef22fcf1..04d07f8cd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) -- Remove schema validation from `lib` folder and use Nextflow nf-validation plugin instead ([#1771](https://github.com/nf-core/tools/pull/1771/)) +- Remove schema validation from `lib` folder and use Nextflow [nf-validation plugin](https://nextflow-io.github.io/nf-validation/) instead ([#1771](https://github.com/nf-core/tools/pull/1771/)) ### Download diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index d6e78fb11f..b6c3336f26 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -25,10 +25,6 @@ class WorkflowMain { // Validate parameters and print summary to screen // public static void initialise(workflow, params, log) { - // Print help to screen if required - if (params.help) { - return - } // Print workflow version and exit on --version if (params.version) { diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 1b5063bb56..abad258c48 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -18,7 +18,6 @@ "exists": true, "mimetype": "text/csv", "pattern": "^\\S+\\.csv$", - "schema": "assets/schema_input.json", "description": "Path to comma-separated file containing information about the samples in the experiment.", "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row.{% if branded %} See [usage docs](https://nf-co.re/{{ short_name }}/usage#samplesheet-input).{% endif %}", "fa_icon": "fas fa-file-csv" @@ -234,6 +233,7 @@ }, "multiqc_config": { "type": "string", + "format": "file-path", "description": "Custom config file to supply to MultiQC.", "fa_icon": "fas fa-cog", "exists": true, From 51882b2559ed06fe7c9321ff7c0601e1ee8240cc Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 09:19:05 +0100 Subject: [PATCH 127/249] Revert "Use specific Nextflow + nf-amazon version to get check if tests pass" This reverts commit d24652af1f7e9dd657bfe0d68b4b637b0c2125b9. --- .github/workflows/create-test-wf.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index d52fa22301..8f3c5fdb47 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,7 +23,8 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.1" + - "22.10.1" + - "latest-everything" steps: - uses: actions/checkout@v3 name: Check out source-code repository @@ -46,7 +47,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - nextflow run nf-core-testpipeline -profile test,docker --outdir ./results -plugins nf-amazon@1.16.2 + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results - name: Upload log file artifact if: ${{ always() }} From 7af5f951512c9d62adeb0e6e0cac3928c4414be6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 9 Jun 2023 08:37:11 +0000 Subject: [PATCH 128/249] do not check if multiqc_config exists --- nf_core/pipeline-template/nextflow_schema.json | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index abad258c48..21d110a476 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -236,7 +236,6 @@ "format": "file-path", "description": "Custom config file to supply to MultiQC.", "fa_icon": "fas fa-cog", - "exists": true, "hidden": true }, "multiqc_logo": { From 225864a6cb71d7bbd21c7d95aa1f419b5898a2b1 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 9 Jun 2023 13:10:52 +0200 Subject: [PATCH 129/249] @alneberg suggested that lastest should not be enforced if other branches exist. --- nf_core/download.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index ee1e6e523f..3886fed0bf 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1258,8 +1258,8 @@ def tidy_tags_and_branches(self): desired_revisions = set(self.revision) # determine what needs pruning - tags_to_remove = {tag for tag in self.repo.tags if tag.name not in desired_revisions} - heads_to_remove = {head for head in self.repo.heads if head.name not in desired_revisions} + tags_to_remove = {tag for tag in self.repo.tags if tag.name not in desired_revisions.union({"latest"})} + heads_to_remove = {head for head in self.repo.heads if head.name not in desired_revisions.union({"latest"})} try: # delete unwanted tags from repository @@ -1276,15 +1276,30 @@ def tidy_tags_and_branches(self): # ensure all desired revisions/branches are available for revision in desired_revisions: - if self.repo.is_valid_object(revision): + if not self.repo.is_valid_object(revision): self.checkout(revision) - # create "latest" branch to ensure at least one branch exists (required for Tower's UI to display revisions correctly) - latest = sorted(desired_revisions, key=StrictVersion)[-1] - self.repo.create_head("latest", latest) - self.checkout(latest) - if self.repo.head.is_detached: - self.repo.head.reset(index=True, working_tree=True) + # no branch exists, but one is required for Tower's UI to display revisions correctly). Thus, "latest" will be created. + if not bool(self.repo.heads): + if self.repo.is_valid_object("latest"): + # "latest" exists as tag but not as branch + self.repo.create_head("latest", "latest") # create a new head for latest + self.checkout("latest") + else: + # "latest" is neither a branch (no heads exist) nor a tag, since is_valid_object() returned False. + # try to determine highest contained version number from desired revisions, to which latest will be pinned. + try: + latest = sorted(desired_revisions, key=StrictVersion, reverse=True)[0] + except ( + ValueError + ): # sorting might fail, because desired revisions may have names that do not comply with semantic version requirements. + # retry sorting using first letters only, this should also sort 3.10 before 3.5 + latest = sorted(desired_revisions, key=lambda x: str(x)[0:4])[0] + finally: + self.repo.create_head("latest", latest) + self.checkout(latest) + if self.repo.head.is_detached: + self.repo.head.reset(index=True, working_tree=True) self.heads = self.repo.heads From 446a6e5fab4761c778bcf008a513b56ede5b9dce Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:11:47 +0100 Subject: [PATCH 130/249] Fix regex for containers --- nf_core/modules/lint/main_nf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index c6ea2ed6c3..5025fa0398 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -268,7 +268,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?", l) + match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)'", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) @@ -280,7 +280,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 - match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)", l) + match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)'", l) if match is not None: docker_tag = match.group(1) self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) From 32fa7707498e6073211a85947d3f70de406520df Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:28:05 +0100 Subject: [PATCH 131/249] Fix parsing of tags so they should match in most situations --- nf_core/modules/lint/main_nf.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 5025fa0398..08645f56bd 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -222,9 +222,6 @@ def check_process_section(self, lines, fix_version, progress_bar): return self.passed.append(("process_exist", "Process definition exists", self.main_nf)) - # Checks that build numbers of bioconda, singularity and docker container are matching - singularity_tag = "singularity" - docker_tag = "docker" bioconda_packages = [] # Process name should be all capital letters @@ -240,11 +237,11 @@ def check_process_section(self, lines, fix_version, progress_bar): # Deprecated enable_conda for i, l in enumerate(lines): url = None - l = l.strip(" \n'\"") + l = l.strip(" \n'\"}:") # Catch preceeding "container " if l.startswith("container"): - l = l.replace("container", "").strip(" \n'\"") + l = l.replace("container", "").strip(": \n'\"}") if _container_type(l) == "conda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] @@ -268,7 +265,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)'", l) + match = re.search(r":([A-Za-z\d\-_.]+)$", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) @@ -280,7 +277,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 - match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)'", l) + match = re.search(r":([A-Za-z\d\-_.]+)$", l) if match is not None: docker_tag = match.group(1) self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) From 90dc179cdd00097c9be1fbad4e587329a5f14eae Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:29:52 +0100 Subject: [PATCH 132/249] Match strip strings for consistency --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 08645f56bd..a31318d935 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -241,7 +241,7 @@ def check_process_section(self, lines, fix_version, progress_bar): # Catch preceeding "container " if l.startswith("container"): - l = l.replace("container", "").strip(": \n'\"}") + l = l.replace("container", "").strip(" \n'\"}:") if _container_type(l) == "conda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] From b6dbc8b8f2e061afde5fb310c21599f47a100610 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:37:10 +0100 Subject: [PATCH 133/249] Revert setting docker_tag and singularity_tag --- nf_core/modules/lint/main_nf.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index a31318d935..85e1f9aa90 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -222,6 +222,11 @@ def check_process_section(self, lines, fix_version, progress_bar): return self.passed.append(("process_exist", "Process definition exists", self.main_nf)) + # Checks that build numbers of bioconda, singularity and docker container are matching + singularity_tag = "singularity" + docker_tag = "docker" + bioconda_packages = [] + bioconda_packages = [] # Process name should be all capital letters From 974565299827e929943c0cdaafe340347840d756 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 9 Jun 2023 15:38:22 +0200 Subject: [PATCH 134/249] Better error handling for failed singularity pullcommands. --- nf_core/download.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 3886fed0bf..2b3ae5894b 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1054,9 +1054,18 @@ def singularity_pull_image(self, container, out_path, cache_path, progress): if lines: # something went wrong with the container retrieval if any("FATAL: " in line for line in lines): - log.info("Singularity container retrieval failed with the following error:") - log.info("".join(lines)) - raise FileNotFoundError(f'The container "{container}" is unavailable.\n{"".join(lines)}') + log.error("Singularity container retrieval failed with the following error:") + log.error("".join(lines)) + if stderr.is_interactive and rich.prompt.Confirm.ask( + f"[blue]Continue downloading the workflow nonetheless?" + ): + log.error( + f'Skipped download of "{container}".\n Please troubleshoot command "{" ".join(singularity_command)}" on an individual basis.' + ) + progress.remove_task(task) + return + else: + raise FileNotFoundError(f'Downloading image "{container}" unfortunately failed.') # Copy cached download if we are using the cache if cache_path: From 8abe1a687f009738d5d197e40e17e2fc859c2fcf Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 9 Jun 2023 17:09:05 +0200 Subject: [PATCH 135/249] Adapt the mechanism to identify the latest version, since distutils module is deprecated. See PEP 632 and PEP 386 for details. --- nf_core/download.py | 48 +++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 2b3ae5894b..347268ea04 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -13,7 +13,6 @@ import tarfile import textwrap from datetime import datetime -from distutils.version import StrictVersion from zipfile import ZipFile import git @@ -23,6 +22,7 @@ import rich import rich.progress from git.exc import GitCommandError, InvalidGitRepositoryError +from pkg_resources import parse_version as VersionParser import nf_core import nf_core.list @@ -886,7 +886,8 @@ def get_singularity_images(self, current_revision=""): # Raise exception if this is not possible log.error("Not able to pull image. Service might be down or internet connection is dead.") raise r - progress.update(task, advance=1) + finally: + progress.update(task, advance=1) def singularity_image_filenames(self, container): """Check Singularity cache for image, copy to destination folder if found. @@ -1054,18 +1055,12 @@ def singularity_pull_image(self, container, out_path, cache_path, progress): if lines: # something went wrong with the container retrieval if any("FATAL: " in line for line in lines): - log.error("Singularity container retrieval failed with the following error:") - log.error("".join(lines)) - if stderr.is_interactive and rich.prompt.Confirm.ask( - f"[blue]Continue downloading the workflow nonetheless?" - ): - log.error( - f'Skipped download of "{container}".\n Please troubleshoot command "{" ".join(singularity_command)}" on an individual basis.' - ) - progress.remove_task(task) - return - else: - raise FileNotFoundError(f'Downloading image "{container}" unfortunately failed.') + log.error(f'[bold red]The singularity image "{container}" could not be pulled:[/]\n\n{"".join(lines)}') + log.error( + f'Skipping failed pull of "{container}". Please troubleshoot the command \n"{" ".join(singularity_command)}"\n\n\n' + ) + progress.remove_task(task) + return # Copy cached download if we are using the cache if cache_path: @@ -1287,6 +1282,9 @@ def tidy_tags_and_branches(self): for revision in desired_revisions: if not self.repo.is_valid_object(revision): self.checkout(revision) + self.repo.create_head(revision, revision) + if self.repo.head.is_detached: + self.repo.head.reset(index=True, working_tree=True) # no branch exists, but one is required for Tower's UI to display revisions correctly). Thus, "latest" will be created. if not bool(self.repo.heads): @@ -1295,18 +1293,16 @@ def tidy_tags_and_branches(self): self.repo.create_head("latest", "latest") # create a new head for latest self.checkout("latest") else: - # "latest" is neither a branch (no heads exist) nor a tag, since is_valid_object() returned False. - # try to determine highest contained version number from desired revisions, to which latest will be pinned. - try: - latest = sorted(desired_revisions, key=StrictVersion, reverse=True)[0] - except ( - ValueError - ): # sorting might fail, because desired revisions may have names that do not comply with semantic version requirements. - # retry sorting using first letters only, this should also sort 3.10 before 3.5 - latest = sorted(desired_revisions, key=lambda x: str(x)[0:4])[0] - finally: - self.repo.create_head("latest", latest) - self.checkout(latest) + # desired revisions may contain arbitrary branch names that do not correspond to valid sematic versioning patterns. + valid_versions = [ + VersionParser(v) + for v in desired_revisions + if re.match(r"\d+\.\d+(?:\.\d+)*(?:[\w\-_])*", v) + ] + # valid versions sorted in ascending order, last will be aliased as "latest". + latest = sorted(valid_versions)[-1] + self.repo.create_head("latest", latest) + self.checkout(latest) if self.repo.head.is_detached: self.repo.head.reset(index=True, working_tree=True) From 56d1a994b08d3a0be581f089afb454abfc2334f0 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 17:36:51 +0100 Subject: [PATCH 136/249] Fewer false positives for docker container names --- nf_core/modules/lint/main_nf.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 85e1f9aa90..6f2d355a7b 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -223,10 +223,8 @@ def check_process_section(self, lines, fix_version, progress_bar): self.passed.append(("process_exist", "Process definition exists", self.main_nf)) # Checks that build numbers of bioconda, singularity and docker container are matching - singularity_tag = "singularity" - docker_tag = "docker" - bioconda_packages = [] - + singularity_tag = None + docker_tag = None bioconda_packages = [] # Process name should be all capital letters @@ -419,7 +417,11 @@ def check_process_section(self, lines, fix_version, progress_bar): else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) - return docker_tag == singularity_tag + # Check if a tag exists at all. If not, return None. + if singularity_tag is None or docker_tag is None: + return None + else: + return docker_tag == singularity_tag def check_process_labels(self, lines): @@ -608,5 +610,5 @@ def _container_type(line): if url_match: return "singularity" return None - if line.count("/") >= 1 and line.count(":") == 1: + if line.count("/") >= 1 and line.count(":") == 1 and line.count(" ") == 0: return "docker" From 1a04dfca1713e86ba503b7bba133b93c36809d55 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 18:10:10 +0100 Subject: [PATCH 137/249] More singularity tag parsing --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 6f2d355a7b..bddf99a06c 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -268,7 +268,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - match = re.search(r":([A-Za-z\d\-_.]+)$", l) + match = re.search(r"(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) From 9379ef885613dd736a0c8cce780d5c491b375f73 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Jun 2023 10:24:17 +0200 Subject: [PATCH 138/249] create .nextflow/nf-core if it doesn't exist --- CHANGELOG.md | 1 + nf_core/refgenie.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2deeef5f66..243c8db136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ - GitPod base image: Always self-update to the latest version of Nextflow. Add [pre-commit](https://pre-commit.com/) dependency. - GitPod configs: Update Nextflow as an init task, init pre-commit in pipeline config. +- Refgenie: Create `nxf_home/nf-core/refgenie_genomes.config` path if it doesn't exist ([]()) # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27] diff --git a/nf_core/refgenie.py b/nf_core/refgenie.py index b666844699..6f09b75532 100644 --- a/nf_core/refgenie.py +++ b/nf_core/refgenie.py @@ -184,7 +184,7 @@ def update_config(rgc): # Save the updated genome config try: - with open(refgenie_genomes_config_file, "w") as fh: + with open(refgenie_genomes_config_file, "w+") as fh: fh.write(refgenie_genomes) log.info(f"Updated nf-core genomes config: {refgenie_genomes_config_file}") except FileNotFoundError: From 87144bfff9386b7c28b37f3687fcff87752ceabb Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 09:26:08 +0100 Subject: [PATCH 139/249] Update comment description of check_process_section where it extracts the container tag --- nf_core/modules/lint/main_nf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index bddf99a06c..d86566a6fd 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -266,8 +266,8 @@ def check_process_section(self, lines, fix_version, progress_bar): ) ) if _container_type(l) == "singularity": - # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 - # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 + # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img -> v1.2.0_cv1 + # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0 -> 0.11.9--0 match = re.search(r"(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l) if match is not None: singularity_tag = match.group(1) @@ -278,8 +278,8 @@ def check_process_section(self, lines, fix_version, progress_bar): url = urlparse(l.split("'")[0]) if _container_type(l) == "docker": - # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 - # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 + # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5 -> 2.7.1--pl526_5 + # e.g. "biocontainers/biocontainers:v1.2.0_cv1 -> v1.2.0_cv1 match = re.search(r":([A-Za-z\d\-_.]+)$", l) if match is not None: docker_tag = match.group(1) From 2c2e2d678c947298d952bb5a484de9ec71923d0d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Jun 2023 10:26:52 +0200 Subject: [PATCH 140/249] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 243c8db136..d729cf46ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ - GitPod base image: Always self-update to the latest version of Nextflow. Add [pre-commit](https://pre-commit.com/) dependency. - GitPod configs: Update Nextflow as an init task, init pre-commit in pipeline config. -- Refgenie: Create `nxf_home/nf-core/refgenie_genomes.config` path if it doesn't exist ([]()) +- Refgenie: Create `nxf_home/nf-core/refgenie_genomes.config` path if it doesn't exist ([#2312](https://github.com/nf-core/tools/pull/2312)) # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27] From 6ef1fa3c5ed205510ca48467cbd56bd0a88d15e3 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 10:06:12 +0100 Subject: [PATCH 141/249] Add period to singularity tag parsing --- nf_core/modules/lint/main_nf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index d86566a6fd..73d1df365a 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -268,7 +268,8 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0 -> 0.11.9--0 - match = re.search(r"(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l) + # Please god let's find a better way to do this than regex + match = re.search(r"(?:[:.])?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) @@ -600,7 +601,7 @@ def _container_type(line): """Returns the container type of a build.""" if line.startswith("conda"): return "conda" - if line.startswith("https://containers") or line.startswith("https://depot"): + if line.startswith("https://") or line.startswith("https://depot"): # Look for a http download URL. # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980 url_regex = ( @@ -610,5 +611,5 @@ def _container_type(line): if url_match: return "singularity" return None - if line.count("/") >= 1 and line.count(":") == 1 and line.count(" ") == 0: + if line.count("/") >= 1 and line.count(":") == 1 and line.count(" ") == 0 and "https://" not in line: return "docker" From 5d00842f9c4ffd97c1a846f0ce95324b5b02b702 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:09:33 +0100 Subject: [PATCH 142/249] Add ability to select custom registry when linting modules Changes: - Adds --registry parameter to nf-core modules lint - Will check container definition via this registry - Default to quay.io --- CHANGELOG.md | 1 + nf_core/__main__.py | 6 +++++- nf_core/modules/lint/__init__.py | 16 +++++++++------- nf_core/modules/lint/main_nf.py | 24 ++++++++++++++++-------- tests/modules/lint.py | 10 ++++++++++ 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6599be574..407e04ed42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) - Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306)) +- Add ability to specify custom registry for linting modules, defaults to quay.io ### Download diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 6d6ded471a..00fba36315 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -807,6 +807,9 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") +@click.option( + "-r", "--registry", type=str, metavar="", default="quay.io", help="Registry to use for containers" +) @click.option("-k", "--key", type=str, metavar="", multiple=True, help="Run only these lint tests") @click.option("-a", "--all", is_flag=True, help="Run on all modules") @click.option("-w", "--fail-warned", is_flag=True, help="Convert warn tests to failures") @@ -821,7 +824,7 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): ) @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") def lint( - ctx, tool, dir, key, all, fail_warned, local, passed, sort_by, fix_version + ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version ): # pylint: disable=redefined-outer-name """ Lint one or more modules in a directory. @@ -846,6 +849,7 @@ def lint( ) module_lint.lint( module=tool, + registry=registry, key=key, all_modules=all, print_results=True, diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 24d673b1c2..f288f17765 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -145,6 +145,7 @@ def get_all_lint_tests(is_pipeline): def lint( self, module=None, + registry="quay.io", key=(), all_modules=False, print_results=True, @@ -227,11 +228,11 @@ def lint( # Lint local modules if local and len(local_modules) > 0: - self.lint_modules(local_modules, local=True, fix_version=fix_version) + self.lint_modules(local_modules, registry=registry, local=True, fix_version=fix_version) # Lint nf-core modules if len(remote_modules) > 0: - self.lint_modules(remote_modules, local=False, fix_version=fix_version) + self.lint_modules(remote_modules, registry=registry, local=False, fix_version=fix_version) if print_results: self._print_results(show_passed=show_passed, sort_by=sort_by) @@ -264,12 +265,13 @@ def filter_tests_by_key(self, key): # If -k supplied, only run these tests self.lint_tests = [k for k in self.lint_tests if k in key] - def lint_modules(self, modules, local=False, fix_version=False): + def lint_modules(self, modules, registry, local=False, fix_version=False): """ Lint a list of modules Args: modules ([NFCoreModule]): A list of module objects + registry (str): The container registry to use. Should be quay.io in most situations. local (boolean): Whether the list consist of local or nf-core modules fix_version (boolean): Fix the module version if a newer version is available """ @@ -290,9 +292,9 @@ def lint_modules(self, modules, local=False, fix_version=False): for mod in modules: progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) - self.lint_module(mod, progress_bar, local=local, fix_version=fix_version) + self.lint_module(mod, progress_bar, registry=registry, local=local, fix_version=fix_version) - def lint_module(self, mod, progress_bar, local=False, fix_version=False): + def lint_module(self, mod, progress_bar, registry, local=False, fix_version=False): """ Perform linting on one module @@ -311,7 +313,7 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False): # Only check the main script in case of a local module if local: - self.main_nf(mod, fix_version, progress_bar) + self.main_nf(mod, fix_version, registry, progress_bar) self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] if not self.fail_warned: @@ -323,7 +325,7 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False): else: for test_name in self.lint_tests: if test_name == "main_nf": - getattr(self, test_name)(mod, fix_version, progress_bar) + getattr(self, test_name)(mod, fix_version, registry, progress_bar) else: getattr(self, test_name)(mod) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 73d1df365a..18d95bd37e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -17,7 +17,7 @@ log = logging.getLogger(__name__) -def main_nf(module_lint_object, module, fix_version, progress_bar): +def main_nf(module_lint_object, module, fix_version, registry, progress_bar): """ Lint a ``main.nf`` module file @@ -121,7 +121,7 @@ def main_nf(module_lint_object, module, fix_version, progress_bar): module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) # Check the process definitions - if check_process_section(module, process_lines, fix_version, progress_bar): + if check_process_section(module, process_lines, registry, fix_version, progress_bar): module.passed.append(("main_nf_container", "Container versions match", module.main_nf)) else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) @@ -209,12 +209,20 @@ def check_when_section(self, lines): self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) -def check_process_section(self, lines, fix_version, progress_bar): - """ - Lint the section of a module between the process definition +def check_process_section(self, lines, registry, fix_version, progress_bar): + """Lint the section of a module between the process definition and the 'input:' definition Specifically checks for correct software versions and containers + + Args: + lines (List[str]): Content of process. + registry (str): Base Docker registry for containers. Typically quay.io. + fix_version (bool): Fix software version + progress_bar (ProgressBar): Progress bar to update. + + Returns: + Optional[bool]: True if singularity and docker containers match, False otherwise. If process definition does not exist, None. """ # Check that we have a process section if len(lines) == 0: @@ -288,7 +296,7 @@ def check_process_section(self, lines, fix_version, progress_bar): else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = None - if l.startswith("quay.io/"): + if l.startswith(registry): l_stripped = re.sub(r"\W+$", "", l) self.failed.append( ( @@ -303,7 +311,7 @@ def check_process_section(self, lines, fix_version, progress_bar): # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04) # If so, add quay.io as default container prefix if l.count("/") == 1 and l.count(":") == 1: - l = "quay.io/" + l + l = "/".join([registry, l]).replace("//", "/") url = urlparse(l.split("'")[0]) # lint double quotes @@ -326,7 +334,7 @@ def check_process_section(self, lines, fix_version, progress_bar): ) # lint more than one container in the same line - if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l): + if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or l.startswith(registry)): self.warned.append( ( "container_links", diff --git a/tests/modules/lint.py b/tests/modules/lint.py index b7aaf610ca..8e692e8cac 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -82,6 +82,16 @@ def test_modules_lint_multiple_remotes(self): assert len(module_lint.warned) >= 0 +def test_modules_lint_registry(self): + """Test linting the TrimGalore! module""" + self.mods_install.install("samtools") + module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) + module_lint.lint(print_results=False, registry="public.ecr.aws", module="samtools") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + def test_modules_lint_patched_modules(self): """ Test creating a patch file and applying it to a new version of the the files From 9b70efb5d1ce2d64ac770bebdff8c4bef827d1ec Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 10:34:01 +0100 Subject: [PATCH 143/249] Quick test update --- tests/modules/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 8e692e8cac..66da0019dc 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -83,7 +83,7 @@ def test_modules_lint_multiple_remotes(self): def test_modules_lint_registry(self): - """Test linting the TrimGalore! module""" + """Test linting the samtools module and alternative registry""" self.mods_install.install("samtools") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) module_lint.lint(print_results=False, registry="public.ecr.aws", module="samtools") From 464f0f75856ecedf3776d45c4d11a2d6ef14d3d5 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 11:04:02 +0100 Subject: [PATCH 144/249] CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 407e04ed42..be0ab9828e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) - Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306)) -- Add ability to specify custom registry for linting modules, defaults to quay.io +- Add ability to specify custom registry for linting modules, defaults to quay.io ([#2313](https://github.com/nf-core/tools/pull/2313)) ### Download From 5e52513f701540c78828d0c769a445e658b4cd21 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 11:04:30 +0100 Subject: [PATCH 145/249] Added 'quay.io' as default for lint_modules --- nf_core/modules/lint/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index f288f17765..ba3b3d18fb 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -265,7 +265,7 @@ def filter_tests_by_key(self, key): # If -k supplied, only run these tests self.lint_tests = [k for k in self.lint_tests if k in key] - def lint_modules(self, modules, registry, local=False, fix_version=False): + def lint_modules(self, modules, registry="quay.io", local=False, fix_version=False): """ Lint a list of modules From 4fe8a043d686199240e16dd393d050626ed846a7 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 13:28:38 +0100 Subject: [PATCH 146/249] Add test for quay.io and public.aws.ecr explicitly for a direct comparison --- tests/modules/lint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 66da0019dc..5b2c250805 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -90,6 +90,10 @@ def test_modules_lint_registry(self): assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 + module_lint.lint(print_results=False, registry="quay.io", module="samtools") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 def test_modules_lint_patched_modules(self): From e5716819de7d959377ad29b048a686a156066475 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Jun 2023 15:49:29 +0200 Subject: [PATCH 147/249] add docs to commit new files before pytest --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index dacb50ebc4..5d5791c109 100644 --- a/README.md +++ b/README.md @@ -941,6 +941,8 @@ before_command: sed 's/1.13a/1.10/g' modules/multiqc/main.nf > modules/multiqc/m To run unit tests of a module that you have installed or the test created by the command [`nf-core modules create-test-yml`](#create-a-module-test-config-file), you can use `nf-core modules test` command. This command runs the tests specified in `modules/tests/software///test.yml` file using [pytest](https://pytest-workflow.readthedocs.io/en/stable/). +> This command uses the pytest argument `--git-aware` to avoid copying the whole `.git` directory and files ignored by `git`. This means that it will only include files listed by `git ls-files`. Remember to **commit your changes** after adding a new module to add the new files to your git index. + You can specify the module name in the form TOOL/SUBTOOL in command line or provide it later by prompts. - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1' -INFO     Updated version in 'nextflow.config' - - version         = '1.0dev' - + version = '1.1' - - + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1' +INFO     Updated version in 'nextflow.config' + - version         = '1.0dev' + + version = '1.1' + + +INFO     Updated version in 'assets/multiqc_config.yml' + - This report has been generated by the <a  +href="https://github.com/nf-core/nextbigthing/1.0dev" target="_blank">nf-core/nextbigthing</a> + + This report has been generated by the <a  +href="https://github.com/nf-core/nextbigthing/1.1" target="_blank">nf-core/nextbigthing</a> + - <a href="https://nf-co.re/nextbigthing/1.0dev/output"  +target="_blank">documentation</a>. + + <a href="https://nf-co.re/nextbigthing/1.1/output" target="_blank">documentation</a>. + + diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index aaae85f91f..c0a7a6db85 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,104 +19,104 @@ font-weight: 700; } - .terminal-1755763245-matrix { + .terminal-2004013614-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1755763245-title { + .terminal-2004013614-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1755763245-r1 { fill: #c5c8c6 } -.terminal-1755763245-r2 { fill: #98a84b } -.terminal-1755763245-r3 { fill: #9a9b99 } -.terminal-1755763245-r4 { fill: #608ab1 } -.terminal-1755763245-r5 { fill: #d0b344 } -.terminal-1755763245-r6 { fill: #98729f } -.terminal-1755763245-r7 { fill: #ff2c7a } -.terminal-1755763245-r8 { fill: #98a84b;font-weight: bold } -.terminal-1755763245-r9 { fill: #1984e9;text-decoration: underline; } + .terminal-2004013614-r1 { fill: #c5c8c6 } +.terminal-2004013614-r2 { fill: #98a84b } +.terminal-2004013614-r3 { fill: #9a9b99 } +.terminal-2004013614-r4 { fill: #608ab1 } +.terminal-2004013614-r5 { fill: #d0b344 } +.terminal-2004013614-r6 { fill: #98729f } +.terminal-2004013614-r7 { fill: #ff2c7a } +.terminal-2004013614-r8 { fill: #98a84b;font-weight: bold } +.terminal-2004013614-r9 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -128,34 +128,34 @@ - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' -INFO     Initialising pipeline git repository                                                        -INFO     Done. Remember to add a remote and push to GitHub:                                          - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      -INFO    !!!!!! IMPORTANT !!!!!! - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' +INFO     Initialising pipeline git repository                                                        +INFO     Done. Remember to add a remote and push to GitHub:                                          + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      +INFO    !!!!!! IMPORTANT !!!!!! + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download-tree.svg b/docs/images/nf-core-download-tree.svg index a06c743fdf..670c1330fe 100644 --- a/docs/images/nf-core-download-tree.svg +++ b/docs/images/nf-core-download-tree.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ tree -L 2 nf-core-rnaseq/ -nf-core-rnaseq/ -├── configs -│   ├── CITATION.cff -│   ├── LICENSE -│   ├── README.md -│   ├── bin -│   ├── conf -│   ├── configtest.nf -│   ├── docs -│   ├── nextflow.config -│   ├── nfcore_custom.config -│   └── pipeline -└── workflow -    ├── CHANGELOG.md -    ├── CITATIONS.md -    ├── CODE_OF_CONDUCT.md -    ├── LICENSE -    ├── README.md -    ├── assets -    ├── bin -    ├── conf -    ├── docs -    ├── lib -    ├── main.nf -    ├── modules -    ├── modules.json -    ├── nextflow.config -    ├── nextflow_schema.json -    ├── subworkflows -    ├── tower.yml -    └── workflows - -14 directories, 16 files + + $ tree -L 2 nf-core-rnaseq/ +nf-core-rnaseq/  [error opening dir] + +0 directories, 0 files diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index a52773fd6d..814b149d02 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -INFO     Saving 'nf-core/rnaseq' -          Pipeline revision: '3.8' -          Pull containers: 'none' -          Output directory: 'nf-core-rnaseq' -INFO     Downloading workflow files from GitHub                                                      -INFO     Downloading centralised configs from GitHub                                                 + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +Usage: nf-core download [OPTIONS] <pipeline name> + +Try 'nf-core download -h' for help. +╭─ Error ──────────────────────────────────────────────────────────────────────────────────────────╮ +│ No such option: -c│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index ddeb0f6e7c..8505f47e74 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-1252240808-matrix { + .terminal-1328852393-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1252240808-title { + .terminal-1328852393-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1252240808-r1 { fill: #c5c8c6 } -.terminal-1252240808-r2 { fill: #98a84b } -.terminal-1252240808-r3 { fill: #9a9b99 } -.terminal-1252240808-r4 { fill: #608ab1 } -.terminal-1252240808-r5 { fill: #d0b344 } -.terminal-1252240808-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1252240808-r7 { fill: #68a0b3;font-weight: bold } + .terminal-1328852393-r1 { fill: #c5c8c6 } +.terminal-1328852393-r2 { fill: #98a84b } +.terminal-1328852393-r3 { fill: #9a9b99 } +.terminal-1328852393-r4 { fill: #608ab1 } +.terminal-1328852393-r5 { fill: #d0b344 } +.terminal-1328852393-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1328852393-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      -         files or profiles                                                                           - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1) + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      +         files or profiles                                                                           + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index de6dff2e78..9a72e4beda 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-2581383703-matrix { + .terminal-2740439576-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2581383703-title { + .terminal-2740439576-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2581383703-r1 { fill: #c5c8c6 } -.terminal-2581383703-r2 { fill: #98a84b } -.terminal-2581383703-r3 { fill: #9a9b99 } -.terminal-2581383703-r4 { fill: #608ab1 } -.terminal-2581383703-r5 { fill: #d0b344 } -.terminal-2581383703-r6 { fill: #68a0b3;font-weight: bold } -.terminal-2581383703-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-2740439576-r1 { fill: #c5c8c6 } +.terminal-2740439576-r2 { fill: #98a84b } +.terminal-2740439576-r3 { fill: #9a9b99 } +.terminal-2740439576-r4 { fill: #608ab1 } +.terminal-2740439576-r5 { fill: #d0b344 } +.terminal-2740439576-r6 { fill: #68a0b3;font-weight: bold } +.terminal-2740439576-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    -INFO     Warning: This tool only prints licence information for the software tools packaged using    -         conda.                                                                                      -INFO     The pipeline may use other software and dependencies not described here.                    -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    +INFO     Warning: This tool only prints licence information for the software tools packaged using    +         conda.                                                                                      +INFO     The pipeline may use other software and dependencies not described here.                    +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index 89e8c43d82..268989f7eb 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Testing pipeline: . - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ -││ -│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ -│doi (after the first release).│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔] 183 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Testing pipeline: . + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ +││ +│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ +│doi (after the first release).│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 3 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│custom/dumpsoftwareversions│modules/nf-core/custom/…│New version available│ +│fastqc│modules/nf-core/fastqc  │New version available│ +│multiqc│modules/nf-core/multiqc │New version available│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 181 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   4 Test Warnings│ +│[✗]   0 Tests Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 0aa93b0d91..414e75d9ba 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + - + - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃Pipeline Name       ┃Stars┃Latest Release┃   Released┃Last Pulled┃Have latest release?┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ smrnaseq             │    49 │          2.2.0 │   yesterday │           - │ -                    │ -│ rnafusion            │    95 │          2.3.4 │  2 days ago │           - │ -                    │ -│ rnaseq               │   604 │         3.11.2 │  3 days ago │           - │ -                    │ -│ dualrnaseq           │    12 │          1.0.0 │ 2 years ago │           - │ -                    │ -│ circrna              │    27 │            dev │           - │           - │ -                    │ -│ lncpipe              │    25 │            dev │           - │           - │ -                    │ -│ scflow               │    19 │            dev │           - │           - │ -                    │ -│ spatialtranscriptom… │    19 │            dev │           - │           - │ -                    │ -└──────────────────────┴───────┴────────────────┴─────────────┴─────────────┴──────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ marsseq              │     3 │          1.0.1 │   4 days ago │           - │ -                   │ +│ scrnaseq             │    87 │          2.3.2 │  3 weeks ago │           - │ -                   │ +│ rnaseq               │   636 │         3.12.0 │  4 weeks ago │           - │ -                   │ +│ smrnaseq             │    50 │          2.2.1 │ 2 months ago │           - │ -                   │ +│ rnafusion            │   105 │          2.3.4 │ 2 months ago │           - │ -                   │ +│ differentialabundan… │    22 │          1.2.0 │ 2 months ago │           - │ -                   │ +│ dualrnaseq           │    13 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    27 │            dev │            - │           - │ -                   │ +│ lncpipe              │    25 │            dev │            - │           - │ -                   │ +│ scflow               │    19 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    24 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index bd24375ed5..06349aa3a1 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-3741580213-matrix { + .terminal-960691040-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3741580213-title { + .terminal-960691040-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3741580213-r1 { fill: #c5c8c6 } -.terminal-3741580213-r2 { fill: #98a84b } -.terminal-3741580213-r3 { fill: #9a9b99 } -.terminal-3741580213-r4 { fill: #608ab1 } -.terminal-3741580213-r5 { fill: #d0b344 } -.terminal-3741580213-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3741580213-r7 { fill: #868887 } -.terminal-3741580213-r8 { fill: #868887;font-style: italic; } + .terminal-960691040-r1 { fill: #c5c8c6 } +.terminal-960691040-r2 { fill: #98a84b } +.terminal-960691040-r3 { fill: #9a9b99 } +.terminal-960691040-r4 { fill: #608ab1 } +.terminal-960691040-r5 { fill: #d0b344 } +.terminal-960691040-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-960691040-r7 { fill: #868887 } +.terminal-960691040-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq              │   604 │         3.11.2 │    3 days ago │           - │ -                   │ -│ sarek               │   235 │          3.1.2 │  4 months ago │           - │ -                   │ -│ chipseq             │   144 │          2.0.0 │  7 months ago │           - │ -                   │ -│ atacseq             │   134 │            2.0 │  5 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   636 │         3.12.0 │  4 weeks ago │           - │ -                   │ +│ sarek                │   252 │          3.2.3 │   1 week ago │           - │ -                   │ +│ chipseq              │   148 │          2.0.0 │ 9 months ago │           - │ -                   │ +│ atacseq              │   141 │            2.0 │ 7 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 3c4a4cd4df..cd47de283f 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-3979640600-matrix { + .terminal-457079933-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3979640600-title { + .terminal-457079933-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3979640600-r1 { fill: #c5c8c6 } -.terminal-3979640600-r2 { fill: #98a84b } -.terminal-3979640600-r3 { fill: #9a9b99 } -.terminal-3979640600-r4 { fill: #608ab1 } -.terminal-3979640600-r5 { fill: #d0b344 } -.terminal-3979640600-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3979640600-r7 { fill: #868887 } -.terminal-3979640600-r8 { fill: #868887;font-style: italic; } + .terminal-457079933-r1 { fill: #c5c8c6 } +.terminal-457079933-r2 { fill: #98a84b } +.terminal-457079933-r3 { fill: #9a9b99 } +.terminal-457079933-r4 { fill: #608ab1 } +.terminal-457079933-r5 { fill: #d0b344 } +.terminal-457079933-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-457079933-r7 { fill: #868887 } +.terminal-457079933-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ funcscan            │    29 │          1.1.0 │  14 hours ago │           - │ -                   │ -│ smrnaseq            │    49 │          2.2.0 │     yesterday │           - │ -                   │ -│ rnafusion           │    95 │          2.3.4 │    2 days ago │           - │ -                   │ -│ rnaseq              │   604 │         3.11.2 │    3 days ago │           - │ -                   │ -│ demultiplex         │    25 │          1.2.0 │    3 days ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ funcscan             │    35 │          1.1.2 │  an hour ago │           - │ -                   │ +│ ampliseq             │   118 │          2.6.1 │   2 days ago │           - │ -                   │ +│ circdna              │    16 │          1.0.4 │   2 days ago │           - │ -                   │ +│ marsseq              │     3 │          1.0.1 │   4 days ago │           - │ -                   │ +│ nanostring           │     4 │          1.1.1 │   6 days ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 0a9094ad26..3bafb91264 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index e49f6dcffa..6ba5712594 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,84 +19,84 @@ font-weight: 700; } - .terminal-1372103280-matrix { + .terminal-3820264502-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1372103280-title { + .terminal-3820264502-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1372103280-r1 { fill: #c5c8c6 } -.terminal-1372103280-r2 { fill: #98a84b } -.terminal-1372103280-r3 { fill: #9a9b99 } -.terminal-1372103280-r4 { fill: #608ab1 } -.terminal-1372103280-r5 { fill: #d0b344 } -.terminal-1372103280-r6 { fill: #ff2c7a } -.terminal-1372103280-r7 { fill: #98729f } + .terminal-3820264502-r1 { fill: #c5c8c6 } +.terminal-3820264502-r2 { fill: #98a84b } +.terminal-3820264502-r3 { fill: #9a9b99 } +.terminal-3820264502-r4 { fill: #608ab1 } +.terminal-3820264502-r5 { fill: #d0b344 } +.terminal-3820264502-r6 { fill: #ff2c7a } +.terminal-3820264502-r7 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108,28 +108,28 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc_paired_end' -INFO     Running 'fastqc' test with command:                                                         -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc_paired_end -c  -./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmpnqfm1ogi -work-dir /tmp/tmpcv36s2sh + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmpk7h2oenp -work-dir /tmp/tmpb21y__ar diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 3a97b353d1..70a03b29a7 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' +INFO     Using Docker container: 'biocontainers/fastqc:0.12.1--hdfd78af_0' +INFO     Using Singularity container:                                                                +'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' +INFO     Created / edited following files:                                                           +           ./modules/nf-core/fastqc/main.nf +           ./modules/nf-core/fastqc/meta.yml +           ./tests/modules/nf-core/fastqc/main.nf +           ./tests/modules/nf-core/fastqc/test.yml +           ./tests/modules/nf-core/fastqc/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index 5bd142d9aa..3a65ab2733 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,163 +19,163 @@ font-weight: 700; } - .terminal-957411833-matrix { + .terminal-1537339898-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-957411833-title { + .terminal-1537339898-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-957411833-r1 { fill: #c5c8c6 } -.terminal-957411833-r2 { fill: #98a84b } -.terminal-957411833-r3 { fill: #9a9b99 } -.terminal-957411833-r4 { fill: #608ab1 } -.terminal-957411833-r5 { fill: #d0b344 } -.terminal-957411833-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-957411833-r7 { fill: #98a84b;font-weight: bold } -.terminal-957411833-r8 { fill: #868887 } -.terminal-957411833-r9 { fill: #d08442 } -.terminal-957411833-r10 { fill: #868887;font-style: italic; } -.terminal-957411833-r11 { fill: #98729f } + .terminal-1537339898-r1 { fill: #c5c8c6 } +.terminal-1537339898-r2 { fill: #98a84b } +.terminal-1537339898-r3 { fill: #9a9b99 } +.terminal-1537339898-r4 { fill: #608ab1 } +.terminal-1537339898-r5 { fill: #d0b344 } +.terminal-1537339898-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1537339898-r7 { fill: #98a84b;font-weight: bold } +.terminal-1537339898-r8 { fill: #868887 } +.terminal-1537339898-r9 { fill: #d08442 } +.terminal-1537339898-r10 { fill: #868887;font-style: italic; } +.terminal-1537339898-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -187,53 +187,53 @@ - + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index f2869ed343..9385dba62f 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-2146405380-matrix { + .terminal-2221378565-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2146405380-title { + .terminal-2221378565-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2146405380-r1 { fill: #c5c8c6 } -.terminal-2146405380-r2 { fill: #98a84b } -.terminal-2146405380-r3 { fill: #9a9b99 } -.terminal-2146405380-r4 { fill: #608ab1 } -.terminal-2146405380-r5 { fill: #d0b344 } + .terminal-2221378565-r1 { fill: #c5c8c6 } +.terminal-2221378565-r2 { fill: #98a84b } +.terminal-2221378565-r3 { fill: #9a9b99 } +.terminal-2221378565-r4 { fill: #608ab1 } +.terminal-2221378565-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Installing 'abacas' -INFO     Use the following statement to include this module:                                         - - include { ABACAS } from '../modules/nf-core/abacas/main'                                            - + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Installing 'abacas' +INFO     Use the following statement to include this module:                                         + + include { ABACAS } from '../modules/nf-core/abacas/main'                                            + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 8f3b419489..08d62c2f3c 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' - -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  24 Tests Passed │ -│[!]   0 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 3128f1caad..cff9db10e3 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-4212330781-matrix { + .terminal-136384798-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4212330781-title { + .terminal-136384798-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4212330781-r1 { fill: #c5c8c6 } -.terminal-4212330781-r2 { fill: #98a84b } -.terminal-4212330781-r3 { fill: #9a9b99 } -.terminal-4212330781-r4 { fill: #608ab1 } -.terminal-4212330781-r5 { fill: #d0b344 } -.terminal-4212330781-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-4212330781-r7 { fill: #868887;font-style: italic; } + .terminal-136384798-r1 { fill: #c5c8c6 } +.terminal-136384798-r2 { fill: #98a84b } +.terminal-136384798-r3 { fill: #9a9b99 } +.terminal-136384798-r4 { fill: #608ab1 } +.terminal-136384798-r5 { fill: #d0b344 } +.terminal-136384798-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-136384798-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                                   - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftwar… │ https://github.co… │ 76cc4938c1f6ea5c7d… │ give execution     │ 2023-04-28 │ -│                     │                    │                     │ permissions to     │            │ -│                     │                    │                     │ dumpsoftwareversi… │            │ -│                     │                    │                     │ (#3347)            │            │ -│ fastqc              │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ -│                     │                    │                     │ syntax for all     │            │ -│                     │                    │                     │ modules (#2654)    │            │ -│ multiqc             │ https://github.co… │ f2d63bd5b68925f98f… │ fix meta.ymls for  │ 2023-04-28 │ -│                     │                    │                     │ dumpsoftware and   │            │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                                   + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftwar… │ https://github.co… │ 76cc4938c1f6ea5c7d… │ give execution     │ 2023-04-28 │ +│                     │                    │                     │ permissions to     │            │ +│                     │                    │                     │ dumpsoftwareversi… │            │ +│                     │                    │                     │ (#3347)            │            │ +│ fastqc              │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ +│                     │                    │                     │ syntax for all     │            │ +│                     │                    │                     │ modules (#2654)    │            │ +│ multiqc             │ https://github.co… │ f2d63bd5b68925f98f… │ fix meta.ymls for  │ 2023-04-28 │ +│                     │                    │                     │ dumpsoftware and   │            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 4faf10450f..1896cb078b 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-3737934755-matrix { + .terminal-3884145572-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3737934755-title { + .terminal-3884145572-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3737934755-r1 { fill: #c5c8c6 } -.terminal-3737934755-r2 { fill: #98a84b } -.terminal-3737934755-r3 { fill: #9a9b99 } -.terminal-3737934755-r4 { fill: #608ab1 } -.terminal-3737934755-r5 { fill: #d0b344 } -.terminal-3737934755-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-3737934755-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-3737934755-r8 { fill: #868887;font-style: italic; } + .terminal-3884145572-r1 { fill: #c5c8c6 } +.terminal-3884145572-r2 { fill: #98a84b } +.terminal-3884145572-r3 { fill: #9a9b99 } +.terminal-3884145572-r4 { fill: #608ab1 } +.terminal-3884145572-r5 { fill: #d0b344 } +.terminal-3884145572-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-3884145572-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3884145572-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Modules available from https://github.com/nf-core/modules.git(master):                     - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                                          ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                                │ -│ abricate/run                                          │ -│ abricate/summary                                      │ -│ adapterremoval                                        │ -│ adapterremovalfixprefix                               │ -│ admixture                                             │ -│ affy/justrma                                          │ -│ agat/convertspgff2gtf                                 │ -│ agat/convertspgxf2gxf                                 │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):                     + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                                          ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                                │ +│ abricate/run                                          │ +│ abricate/summary                                      │ +│ adapterremoval                                        │ +│ adapterremovalfixprefix                               │ +│ admixture                                             │ +│ affy/justrma                                          │ +│ agat/convertspgff2gtf                                 │ +│ agat/convertspgxf2gxf                                 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index b0d8a66a95..b5cf52459b 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-277091844-matrix { + .terminal-314906117-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-277091844-title { + .terminal-314906117-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-277091844-r1 { fill: #c5c8c6 } -.terminal-277091844-r2 { fill: #98a84b } -.terminal-277091844-r3 { fill: #9a9b99 } -.terminal-277091844-r4 { fill: #608ab1 } -.terminal-277091844-r5 { fill: #d0b344 } -.terminal-277091844-r6 { fill: #cc555a;font-weight: bold } + .terminal-314906117-r1 { fill: #c5c8c6 } +.terminal-314906117-r2 { fill: #98a84b } +.terminal-314906117-r3 { fill: #9a9b99 } +.terminal-314906117-r4 { fill: #608ab1 } +.terminal-314906117-r5 { fill: #d0b344 } +.terminal-314906117-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute                           + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute                           diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index fb1d3591ff..9ccbb37c09 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-3673042259-matrix { + .terminal-3716230484-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3673042259-title { + .terminal-3716230484-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3673042259-r1 { fill: #c5c8c6 } -.terminal-3673042259-r2 { fill: #98a84b } -.terminal-3673042259-r3 { fill: #9a9b99 } -.terminal-3673042259-r4 { fill: #608ab1 } -.terminal-3673042259-r5 { fill: #d0b344 } + .terminal-3716230484-r1 { fill: #c5c8c6 } +.terminal-3716230484-r2 { fill: #98a84b } +.terminal-3716230484-r3 { fill: #9a9b99 } +.terminal-3716230484-r4 { fill: #608ab1 } +.terminal-3716230484-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Removed files for 'abacas' and it's dependencies 'abacas'.                                  + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Removed files for 'abacas' and it's dependencies 'abacas'.                                  diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 83f6535612..3c6eeaca9e 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view' + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +WARNING  You have uncommitted changes. Make sure to commit last changes before running the tests.    +INFO     Running pytest for module 'samtools/view' diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index a41cbbeec4..b4766f98c4 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-3947254567-matrix { + .terminal-2020623448-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3947254567-title { + .terminal-2020623448-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3947254567-r1 { fill: #c5c8c6 } -.terminal-3947254567-r2 { fill: #98a84b } -.terminal-3947254567-r3 { fill: #9a9b99 } -.terminal-3947254567-r4 { fill: #608ab1 } -.terminal-3947254567-r5 { fill: #d0b344 } + .terminal-2020623448-r1 { fill: #c5c8c6 } +.terminal-2020623448-r2 { fill: #98a84b } +.terminal-2020623448-r3 { fill: #9a9b99 } +.terminal-2020623448-r4 { fill: #608ab1 } +.terminal-2020623448-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                              -INFO    'modules/nf-core/custom/dumpsoftwareversions' is already up to date                         -INFO    'modules/nf-core/fastqc' is already up to date                                              -INFO    'modules/nf-core/multiqc' is already up to date                                             -INFO     Updates complete ✨                                                                         + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                              +INFO     Updating 'nf-core/custom/dumpsoftwareversions' +INFO     Updating 'nf-core/fastqc' +INFO     Updating 'nf-core/multiqc' +INFO     Updates complete ✨                                                                         diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 5fa9746d5a..21bb725c6d 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-3395078802-matrix { + .terminal-3584346773-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3395078802-title { + .terminal-3584346773-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3395078802-r1 { fill: #c5c8c6 } -.terminal-3395078802-r2 { fill: #98a84b } -.terminal-3395078802-r3 { fill: #9a9b99 } -.terminal-3395078802-r4 { fill: #608ab1 } -.terminal-3395078802-r5 { fill: #d0b344 } -.terminal-3395078802-r6 { fill: #98a84b;font-weight: bold } -.terminal-3395078802-r7 { fill: #868887;font-weight: bold } -.terminal-3395078802-r8 { fill: #868887 } -.terminal-3395078802-r9 { fill: #4e707b;font-weight: bold } -.terminal-3395078802-r10 { fill: #68a0b3;font-weight: bold } + .terminal-3584346773-r1 { fill: #c5c8c6 } +.terminal-3584346773-r2 { fill: #98a84b } +.terminal-3584346773-r3 { fill: #9a9b99 } +.terminal-3584346773-r4 { fill: #608ab1 } +.terminal-3584346773-r5 { fill: #d0b344 } +.terminal-3584346773-r6 { fill: #98a84b;font-weight: bold } +.terminal-3584346773-r7 { fill: #868887;font-weight: bold } +.terminal-3584346773-r8 { fill: #868887 } +.terminal-3584346773-r9 { fill: #4e707b;font-weight: bold } +.terminal-3584346773-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 30 params) -INFO     Writing schema with 31 params: './nextflow_schema.json' + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 31 params) +INFO     Writing schema with 32 params: './nextflow_schema.json' diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index a5b8a94fd3..ec9dcbb236 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - - $ nf-core schema lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 31 params) + + $ nf-core schema lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +WARNING [!] Default parameter 'outdir' is empty or null. It is advisable to remove the default from +the schema +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 32 params) diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 0ec7e246aa..14ea6ff059 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 93 params) -INFO    [✓] Input parameters look valid + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ +│/opt/hostedtoolcache/Python/3.11.4/x64/bin/nf-core:8 in <module>│ +││ +│5 fromnf_core.__main__import run_nf_core                                                     │ +│6 if__name__ == '__main__':                                                                   │ +│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ +│❱ 8 │   sys.exit(run_nf_core())                                                                  │ +│9 │ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/__main__.py:110 in   │ +│run_nf_core│ +││ +│ 107 │   │   │   log.debug(f"Could not check latest version: {e}")                             │ +│ 108 │   │   stderr.print("\n")                                                                │ +│ 109 │   # Launch the click cli│ +│❱  110 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ +│ 111 │ +│ 112 │ +│ 113 @click.group(context_settings=dict(help_option_names=["-h", "--help"]))                   │ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1130 in        │ +│__call__│ +││ +│1127 │   │ +│1128 │   def__call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any:                           │ +│1129 │   │   """Alias for :meth:`main`."""│ +│❱ 1130 │   │   returnself.main(*args, **kwargs)                                                 │ +│1131 │ +│1132 │ +│1133 classCommand(BaseCommand):                                                               │ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/rich_click/rich_group.py:21│ +│ in main│ +││ +│18 │   │ +│19 │   defmain(self, *args, standalone_mode: bool = True, **kwargs):                          │ +│20 │   │   try:                                                                                │ +│❱ 21 │   │   │   rv = super().main(*args, standalone_mode=False, **kwargs)                       │ +│22 │   │   │   ifnot standalone_mode:                                                         │ +│23 │   │   │   │   return rv                                                                   │ +│24 │   │   except click.ClickException as e:                                                   │ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1055 in main│ +││ +│1052 │   │   try:                                                                              │ +│1053 │   │   │   try:                                                                          │ +│1054 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                  │ +│❱ 1055 │   │   │   │   │   rv = self.invoke(ctx)                                                 │ +│1056 │   │   │   │   │   ifnot standalone_mode:                                               │ +│1057 │   │   │   │   │   │   return rv                                                         │ +│1058 │   │   │   │   │   # it's not safe to `ctx.exit(rv)` here!│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ +││ +│1654 │   │   │   │   super().invoke(ctx)                                                       │ +│1655 │   │   │   │   sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)                    │ +│1656 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1658 │   │   │ +│1659 │   │   # In chain mode we create the contexts step by step, but after the│ +│1660 │   │   # base command has been invoked.  Because at that point we do not│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ +││ +│1654 │   │   │   │   super().invoke(ctx)                                                       │ +│1655 │   │   │   │   sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)                    │ +│1656 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1658 │   │   │ +│1659 │   │   # In chain mode we create the contexts step by step, but after the│ +│1660 │   │   # base command has been invoked.  Because at that point we do not│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1404 in invoke│ +││ +│1401 │   │   │   echo(style(message, fg="red"), err=True)                                      │ +│1402 │   │   │ +│1403 │   │   ifself.callback isnotNone:                                                     │ +│❱ 1404 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ +│1405 │   │ +│1406 │   defshell_complete(self, ctx: Context, incomplete: str) -> t.List["CompletionItem"]:  │ +│1407 │   │   """Return a list of completions for the incomplete value. Looks│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:760 in invoke│ +││ +│ 757 │   │   │ +│ 758 │   │   with augment_usage_errors(__self):                                                │ +│ 759 │   │   │   with ctx:                                                                     │ +│❱  760 │   │   │   │   return __callback(*args, **kwargs)                                        │ +│ 761 │   │ +│ 762 │   defforward(                                                                          │ +│ 763 │   │   __self, __cmd: "Command", *args: t.Any, **kwargs: t.Any  # noqa: B902│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/__main__.py:1369 in  │ +│validate│ +││ +│1366 │   │ +│1367 │   schema_obj = PipelineSchema()                                                         │ +│1368 │   try:                                                                                  │ +│❱ 1369 │   │   schema_obj.get_schema_path(pipeline)                                              │ +│1370 │   │   # Load and check schema│ +│1371 │   │   schema_obj.load_lint_schema()                                                     │ +│1372 │   exceptAssertionErroras e:                                                           │ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/schema.py:66 in      │ +│get_schema_path│ +││ +│ 63 │   │   │ +│ 64 │   │   # Path does not exist - assume a name of a remote workflow│ +│ 65 │   │   elifnot local_only:                                                               │ +│❱  66 │   │   │   self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision)         │ +│ 67 │   │   │   self.schema_filename = os.path.join(self.pipeline_dir, "nextflow_schema.json│ +│ 68 │   │   │ +│ 69 │   │   # Only looking for local paths, overwrite with None to be safe│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/list.py:52 in        │ +│get_local_wf│ +││ +│ 49 │   │   workflow = f"nf-core/{workflow}"│ +│ 50 │   │ +│ 51 │   wfs = Workflows()                                                                      │ +│❱  52 │   wfs.get_local_nf_workflows()                                                           │ +│ 53 │   for wf in wfs.local_workflows:                                                         │ +│ 54 │   │   if workflow == wf.full_name:                                                       │ +│ 55 │   │   │   if revision isNoneor revision == wf.commit_sha or revision == wf.branch or│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/list.py:141 in       │ +│get_local_nf_workflows│ +││ +│138 │   │   # Find additional information about each workflow by checking its git history│ +│139 │   │   log.debug(f"Fetching extra info about {len(self.local_workflows)} local workflow│ +│140 │   │   for wf inself.local_workflows:                                                    │ +│❱ 141 │   │   │   wf.get_local_nf_workflow_details()                                             │ +│142 │   │ +│143 │   defcompare_remote_local(self):                                                        │ +│144 │   │   """Matches local to remote workflows.│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/list.py:356 in       │ +│get_local_nf_workflow_details│ +││ +│353 │   │   │   log.debug(f"Pulling git info from {self.local_path}")                          │ +│354 │   │   │   try:                                                                           │ +│355 │   │   │   │   repo = git.Repo(self.local_path)                                           │ +│❱ 356 │   │   │   │   self.commit_sha = str(repo.head.commit.hexsha)                             │ +│357 │   │   │   │   self.remote_url = str(repo.remotes.origin.url)                             │ +│358 │   │   │   │   self.last_pull = os.stat(os.path.join(self.local_path, ".git", "FETCH_HE│ +│359 │   │   │   │   self.last_pull_date = datetime.fromtimestamp(self.last_pull).strftime("%│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:226 in  │ +│_get_commit│ +││ +│223 │   │   :return:│ +│224 │   │   │   Commit object we point to, works for detached and non-detached│ +│225 │   │   │   SymbolicReferences. The symbolic reference will be dereferenced recursively.│ +│❱ 226 │   │   obj = self._get_object()                                                           │ +│227 │   │   if obj.type == "tag":                                                              │ +│228 │   │   │   obj = obj.object                                                               │ +│229 │   │   # END dereference tag│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:219 in  │ +│_get_object│ +││ +│216 │   │   │   always point to the actual object as it gets re-created on each query"""│ +│217 │   │   # have to be dynamic here as we may be a tag which can point to anything│ +│218 │   │   # Our path will be resolved to the hexsha which will be used accordingly│ +│❱ 219 │   │   return Object.new_from_sha(self.repo, hex_to_bin(self.dereference_recursive(self│ +│220 │   │ +│221 │   def_get_commit(self) -> "Commit":                                                     │ +│222 │   │   """│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:159 in  │ +│dereference_recursive│ +││ +│156 │   │   :param repo: the repository containing the reference at ref_path"""│ +│157 │   │   │ +│158 │   │   whileTrue:                                                                        │ +│❱ 159 │   │   │   hexsha, ref_path = cls._get_ref_info(repo, ref_path)                           │ +│160 │   │   │   if hexsha isnotNone:                                                         │ +│161 │   │   │   │   return hexsha                                                              │ +│162 │   │   # END recursive dereferencing│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:210 in  │ +│_get_ref_info│ +││ +│207 │   │   """Return: (str(sha), str(target_ref_path)) if available, the sha the file at│ +│208 │   │   rela_path points to, or None. target_ref_path is the reference we│ +│209 │   │   point to, or None"""│ +│❱ 210 │   │   returncls._get_ref_info_helper(repo, ref_path)                                    │ +│211 │   │ +│212 │   def_get_object(self) -> Commit_ish:                                                   │ +│213 │   │   """│ +││ +│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:193 in  │ +│_get_ref_info_helper│ +││ +│190 │   │   │   # END for each packed ref│ +│191 │   │   # END handle packed refs│ +│192 │   │   if tokens isNone:                                                                 │ +│❱ 193 │   │   │   raiseValueError("Reference at %r does not exist" % ref_path)                  │ +│194 │   │   │ +│195 │   │   # is it a reference ?│ +│196 │   │   if tokens[0] == "ref:":                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +ValueError: Reference at 'refs/heads/master' does not exist diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index 074b32dd15..07bc2df151 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -19,84 +19,84 @@ font-weight: 700; } - .terminal-1209424158-matrix { + .terminal-1000036394-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1209424158-title { + .terminal-1000036394-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1209424158-r1 { fill: #c5c8c6 } -.terminal-1209424158-r2 { fill: #98a84b } -.terminal-1209424158-r3 { fill: #9a9b99 } -.terminal-1209424158-r4 { fill: #608ab1 } -.terminal-1209424158-r5 { fill: #d0b344 } -.terminal-1209424158-r6 { fill: #ff2c7a } -.terminal-1209424158-r7 { fill: #98729f } + .terminal-1000036394-r1 { fill: #c5c8c6 } +.terminal-1000036394-r2 { fill: #98a84b } +.terminal-1000036394-r3 { fill: #9a9b99 } +.terminal-1000036394-r4 { fill: #608ab1 } +.terminal-1000036394-r5 { fill: #d0b344 } +.terminal-1000036394-r6 { fill: #ff2c7a } +.terminal-1000036394-r7 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108,28 +108,28 @@ - + - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Looking for test workflow entry points:                                                     -'tests/subworkflows/nf-core/bam_stats_samtools/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_bam_stats_samtools' -INFO     Running 'bam_stats_samtools' test with command:                                             -nextflow run ./tests/subworkflows/nf-core/bam_stats_samtools -entry test_bam_stats_samtools --c ./tests/config/nextflow.config --outdir /tmp/tmp4iszml15 -work-dir /tmp/tmp1r4spwkd + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Looking for test workflow entry points:                                                     +'tests/subworkflows/nf-core/bam_stats_samtools/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_bam_stats_samtools' +INFO     Running 'bam_stats_samtools' test with command:                                             +nextflow run ./tests/subworkflows/nf-core/bam_stats_samtools -entry test_bam_stats_samtools +-c ./tests/config/nextflow.config --outdir /tmp/tmp05wh1761 -work-dir /tmp/tmpwg_j73w9 diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index 215fce11c0..df2311a2fd 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -19,94 +19,94 @@ font-weight: 700; } - .terminal-3514492232-matrix { + .terminal-3762021705-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3514492232-title { + .terminal-3762021705-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3514492232-r1 { fill: #c5c8c6 } -.terminal-3514492232-r2 { fill: #98a84b } -.terminal-3514492232-r3 { fill: #9a9b99 } -.terminal-3514492232-r4 { fill: #608ab1 } -.terminal-3514492232-r5 { fill: #d0b344 } -.terminal-3514492232-r6 { fill: #68a0b3;font-weight: bold } -.terminal-3514492232-r7 { fill: #98729f } -.terminal-3514492232-r8 { fill: #ff2c7a } + .terminal-3762021705-r1 { fill: #c5c8c6 } +.terminal-3762021705-r2 { fill: #98a84b } +.terminal-3762021705-r3 { fill: #9a9b99 } +.terminal-3762021705-r4 { fill: #608ab1 } +.terminal-3762021705-r5 { fill: #d0b344 } +.terminal-3762021705-r6 { fill: #68a0b3;font-weight: bold } +.terminal-3762021705-r7 { fill: #98729f } +.terminal-3762021705-r8 { fill: #ff2c7a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -118,31 +118,31 @@ - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Created / edited following files:                                                           -           ./subworkflows/nf-core/bam_stats_samtools/main.nf -           ./subworkflows/nf-core/bam_stats_samtools/meta.yml -           ./tests/subworkflows/nf-core/bam_stats_samtools/main.nf -           ./tests/subworkflows/nf-core/bam_stats_samtools/test.yml -           ./tests/subworkflows/nf-core/bam_stats_samtools/nextflow.config -           ./tests/config/pytest_modules.yml + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Created / edited following files:                                                           +           ./subworkflows/nf-core/bam_stats_samtools/main.nf +           ./subworkflows/nf-core/bam_stats_samtools/meta.yml +           ./tests/subworkflows/nf-core/bam_stats_samtools/main.nf +           ./tests/subworkflows/nf-core/bam_stats_samtools/test.yml +           ./tests/subworkflows/nf-core/bam_stats_samtools/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-subworkflows-info.svg b/docs/images/nf-core-subworkflows-info.svg index bace90622f..ce903ed11b 100644 --- a/docs/images/nf-core-subworkflows-info.svg +++ b/docs/images/nf-core-subworkflows-info.svg @@ -19,304 +19,304 @@ font-weight: 700; } - .terminal-4158441153-matrix { + .terminal-1571342018-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4158441153-title { + .terminal-1571342018-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4158441153-r1 { fill: #c5c8c6 } -.terminal-4158441153-r2 { fill: #98a84b } -.terminal-4158441153-r3 { fill: #9a9b99 } -.terminal-4158441153-r4 { fill: #608ab1 } -.terminal-4158441153-r5 { fill: #d0b344 } -.terminal-4158441153-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-4158441153-r7 { fill: #98a84b;font-weight: bold } -.terminal-4158441153-r8 { fill: #868887 } -.terminal-4158441153-r9 { fill: #d08442 } -.terminal-4158441153-r10 { fill: #868887;font-style: italic; } -.terminal-4158441153-r11 { fill: #98729f } + .terminal-1571342018-r1 { fill: #c5c8c6 } +.terminal-1571342018-r2 { fill: #98a84b } +.terminal-1571342018-r3 { fill: #9a9b99 } +.terminal-1571342018-r4 { fill: #608ab1 } +.terminal-1571342018-r5 { fill: #d0b344 } +.terminal-1571342018-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1571342018-r7 { fill: #98a84b;font-weight: bold } +.terminal-1571342018-r8 { fill: #868887 } +.terminal-1571342018-r9 { fill: #d08442 } +.terminal-1571342018-r10 { fill: #868887;font-style: italic; } +.terminal-1571342018-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -328,100 +328,100 @@ - + - - $ nf-core subworkflows info bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -╭─ Subworkflow: bam_rseqc  ────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 📖 Description: Subworkflow to run multiple commands in the RSeqC package                        │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                       ╷                                                                   ╷         -📥 Inputs             │Description                                                        │Pattern -╺━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━╸ - meta  (map)          │Groovy Map containing sample information e.g. [ id:'test',         │ -                       │single_end:false ]                                                 │ -╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ - bam  (file)          │BAM file to calculate statistics                                   │*.{bam} -╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ - bai  (file)          │Index for input BAM file                                           │*.{bai} -╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ - bed  (file)          │BED file for the reference gene model                              │*.{bed} -╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ - rseqc_modules  (list)│List of rseqc modules to run e.g. [ 'bam_stat', 'infer_experiment' │ -                       │]                                                                  │ -                       ╵                                                                   ╵         -                                     ╷                                   ╷                           -📤 Outputs                          │Description                        │                  Pattern -╺━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━╸ - bamstat_txt  (file)                │bam statistics report              │           *.bam_stat.txt -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - innerdistance_distance  (file)     │the inner distances                │     *.inner_distance.txt -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - innerdistance_freq  (file)         │frequencies of different insert    │*.inner_distance_freq.txt -                                     │sizes                              │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - innerdistance_mean  (file)         │mean/median values of inner        │*.inner_distance_mean.txt -                                     │distances                          │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - innerdistance_pdf  (file)          │distribution plot of inner         │*.inner_distance_plot.pdf -                                     │distances                          │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - innerdistance_rscript  (file)      │script to reproduce the plot       │  *.inner_distance_plot.R -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - inferexperiment_txt  (file)        │infer_experiment results report    │   *.infer_experiment.txt -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - junctionannotation_bed  (file)     │bed file of annotated junctions    │           *.junction.bed -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - junctionannotation_interact_bed   │Interact bed file                  │           *.Interact.bed -(file)                              │                                   │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - junctionannotation_xls  (file)     │xls file with junction information │                    *.xls -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - junctionannotation_pdf  (file)     │junction plot                      │           *.junction.pdf -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - junctionannotation_events_pdf     │events plot                        │             *.events.pdf -(file)                              │                                   │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - junctionannotation_rscript  (file) │Rscript to reproduce the plots     │                      *.r -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - junctionannotation_log  (file)     │Log file generated by tool         │                    *.log -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - junctionsaturation_pdf  (file)     │Junction saturation report         │                    *.pdf -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - junctionsaturation_rscript  (file) │Junction saturation R-script       │                      *.r -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - readdistribution_txt  (file)       │the read distribution report       │  *.read_distribution.txt -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - readduplication_seq_xls  (file)    │Read duplication rate determined   │         *seq.DupRate.xls -                                     │from mapping position of read      │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - readduplication_pos_xls  (file)    │Read duplication rate determined   │         *pos.DupRate.xls -                                     │from sequence of read              │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - readduplication_pdf  (file)        │plot of duplication rate           │                    *.pdf -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - readduplication_rscript  (file)    │script to reproduce the plot       │                      *.R -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - tin_txt  (file)                    │TXT file containing tin.py results │                    *.txt -                                     │summary                            │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - versions  (file)                   │File containing software versions  │             versions.yml -                                     ╵                                   ╵                           - - 💻  Installation command: nf-core subworkflows install bam_rseqc - + + $ nf-core subworkflows info bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +╭─ Subworkflow: bam_rseqc  ────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 📖 Description: Subworkflow to run multiple commands in the RSeqC package                        │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                       ╷                                                                   ╷         +📥 Inputs             │Description                                                        │Pattern +╺━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━╸ + meta  (map)          │Groovy Map containing sample information e.g. [ id:'test',         │ +                       │single_end:false ]                                                 │ +╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ + bam  (file)          │BAM file to calculate statistics                                   │*.{bam} +╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ + bai  (file)          │Index for input BAM file                                           │*.{bai} +╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ + bed  (file)          │BED file for the reference gene model                              │*.{bed} +╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ + rseqc_modules  (list)│List of rseqc modules to run e.g. [ 'bam_stat', 'infer_experiment' │ +                       │]                                                                  │ +                       ╵                                                                   ╵         +                                     ╷                                   ╷                           +📤 Outputs                          │Description                        │                  Pattern +╺━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━╸ + bamstat_txt  (file)                │bam statistics report              │           *.bam_stat.txt +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_distance  (file)     │the inner distances                │     *.inner_distance.txt +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_freq  (file)         │frequencies of different insert    │*.inner_distance_freq.txt +                                     │sizes                              │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_mean  (file)         │mean/median values of inner        │*.inner_distance_mean.txt +                                     │distances                          │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_pdf  (file)          │distribution plot of inner         │*.inner_distance_plot.pdf +                                     │distances                          │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_rscript  (file)      │script to reproduce the plot       │  *.inner_distance_plot.R +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + inferexperiment_txt  (file)        │infer_experiment results report    │   *.infer_experiment.txt +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_bed  (file)     │bed file of annotated junctions    │           *.junction.bed +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_interact_bed   │Interact bed file                  │           *.Interact.bed +(file)                              │                                   │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_xls  (file)     │xls file with junction information │                    *.xls +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_pdf  (file)     │junction plot                      │           *.junction.pdf +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_events_pdf     │events plot                        │             *.events.pdf +(file)                              │                                   │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_rscript  (file) │Rscript to reproduce the plots     │                      *.r +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_log  (file)     │Log file generated by tool         │                    *.log +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionsaturation_pdf  (file)     │Junction saturation report         │                    *.pdf +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionsaturation_rscript  (file) │Junction saturation R-script       │                      *.r +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readdistribution_txt  (file)       │the read distribution report       │  *.read_distribution.txt +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readduplication_seq_xls  (file)    │Read duplication rate determined   │         *seq.DupRate.xls +                                     │from mapping position of read      │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readduplication_pos_xls  (file)    │Read duplication rate determined   │         *pos.DupRate.xls +                                     │from sequence of read              │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readduplication_pdf  (file)        │plot of duplication rate           │                    *.pdf +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readduplication_rscript  (file)    │script to reproduce the plot       │                      *.R +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + tin_txt  (file)                    │TXT file containing tin.py results │                    *.txt +                                     │summary                            │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + versions  (file)                   │File containing software versions  │             versions.yml +                                     ╵                                   ╵                           + + 💻  Installation command: nf-core subworkflows install bam_rseqc + diff --git a/docs/images/nf-core-subworkflows-install.svg b/docs/images/nf-core-subworkflows-install.svg index 7363919195..7dfa30a301 100644 --- a/docs/images/nf-core-subworkflows-install.svg +++ b/docs/images/nf-core-subworkflows-install.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-1929907304-matrix { + .terminal-1967000681-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1929907304-title { + .terminal-1967000681-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1929907304-r1 { fill: #c5c8c6 } -.terminal-1929907304-r2 { fill: #98a84b } -.terminal-1929907304-r3 { fill: #9a9b99 } -.terminal-1929907304-r4 { fill: #608ab1 } -.terminal-1929907304-r5 { fill: #d0b344 } + .terminal-1967000681-r1 { fill: #c5c8c6 } +.terminal-1967000681-r2 { fill: #98a84b } +.terminal-1967000681-r3 { fill: #9a9b99 } +.terminal-1967000681-r4 { fill: #608ab1 } +.terminal-1967000681-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core subworkflows install bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Installing 'bam_rseqc' + + $ nf-core subworkflows install bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Installing 'bam_rseqc' diff --git a/docs/images/nf-core-subworkflows-list-local.svg b/docs/images/nf-core-subworkflows-list-local.svg index dc2836367d..cd62723604 100644 --- a/docs/images/nf-core-subworkflows-list-local.svg +++ b/docs/images/nf-core-subworkflows-list-local.svg @@ -19,67 +19,67 @@ font-weight: 700; } - .terminal-1394611191-matrix { + .terminal-1433277432-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1394611191-title { + .terminal-1433277432-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1394611191-r1 { fill: #c5c8c6 } -.terminal-1394611191-r2 { fill: #98a84b } -.terminal-1394611191-r3 { fill: #9a9b99 } -.terminal-1394611191-r4 { fill: #608ab1 } -.terminal-1394611191-r5 { fill: #d0b344 } + .terminal-1433277432-r1 { fill: #c5c8c6 } +.terminal-1433277432-r2 { fill: #98a84b } +.terminal-1433277432-r3 { fill: #9a9b99 } +.terminal-1433277432-r4 { fill: #608ab1 } +.terminal-1433277432-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -91,23 +91,23 @@ - + - - $ nf-core subworkflows list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     No nf-core subworkflows found in '.' - + + $ nf-core subworkflows list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     No nf-core subworkflows found in '.' + diff --git a/docs/images/nf-core-subworkflows-list-remote.svg b/docs/images/nf-core-subworkflows-list-remote.svg index 025ee22a18..a14e7b006d 100644 --- a/docs/images/nf-core-subworkflows-list-remote.svg +++ b/docs/images/nf-core-subworkflows-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-1279347052-matrix { + .terminal-1417169261-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1279347052-title { + .terminal-1417169261-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1279347052-r1 { fill: #c5c8c6 } -.terminal-1279347052-r2 { fill: #98a84b } -.terminal-1279347052-r3 { fill: #9a9b99 } -.terminal-1279347052-r4 { fill: #608ab1 } -.terminal-1279347052-r5 { fill: #d0b344 } -.terminal-1279347052-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-1279347052-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-1279347052-r8 { fill: #868887;font-style: italic; } + .terminal-1417169261-r1 { fill: #c5c8c6 } +.terminal-1417169261-r2 { fill: #98a84b } +.terminal-1417169261-r3 { fill: #9a9b99 } +.terminal-1417169261-r4 { fill: #608ab1 } +.terminal-1417169261-r5 { fill: #d0b344 } +.terminal-1417169261-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-1417169261-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-1417169261-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core subworkflows list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Subworkflow Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ bam_create_som_pon_gatk                       │ -│ bam_dedup_stats_samtools_umitools             │ -│ bam_docounts_contamination_angsd              │ -│ bam_markduplicates_picard                     │ -│ bam_ngscheckmate                              │ -│ bam_qc_picard                                 │ -│ bam_rseqc                                     │ -│ bam_sort_stats_samtools                       │ -│ bam_split_by_region                           │ -[..truncated..] + + $ nf-core subworkflows list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Subworkflow Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ bam_create_som_pon_gatk                       │ +│ bam_dedup_stats_samtools_umitools             │ +│ bam_docounts_contamination_angsd              │ +│ bam_markduplicates_picard                     │ +│ bam_ngscheckmate                              │ +│ bam_qc_picard                                 │ +│ bam_rseqc                                     │ +│ bam_sort_stats_samtools                       │ +│ bam_split_by_region                           │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 637ad9101c..040c8d999b 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Removed files for 'rseqc/bamstat' and it's dependencies 'rseqc/bamstat'.                    -INFO     Removed files for 'rseqc/inferexperiment' and it's dependencies 'rseqc/inferexperiment'.    -INFO     Removed files for 'rseqc/innerdistance' and it's dependencies 'rseqc/innerdistance'.        -INFO     Removed files for 'bam_rseqc' and it's dependencies 'bam_rseqc, rseqc_bamstat,  -rseqc_inferexperiment, rseqc_innerdistance'.                                                + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Removed files for 'rseqc/bamstat' and it's dependencies 'rseqc/bamstat'.                    +INFO     Removed files for 'rseqc/inferexperiment' and it's dependencies 'rseqc/inferexperiment'.    +INFO     Removed files for 'rseqc/innerdistance' and it's dependencies 'rseqc/innerdistance'.        +INFO     Removed files for 'rseqc/junctionannotation' and it's dependencies                          +'rseqc/junctionannotation'.                                                                 +INFO     Removed files for 'bam_rseqc' and it's dependencies 'bam_rseqc, rseqc_bamstat,  +rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation'.                      diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index f9ba18cc79..459502886a 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -──────────────────────────────────────────── bam_rseqc ───────────────────────────────────────────── -INFO     Running pytest for subworkflow 'bam_rseqc' + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +──────────────────────────────────────────── bam_rseqc ───────────────────────────────────────────── +WARNING  You have uncommitted changes. Make sure to commit last changes before running the tests.    +INFO     Running pytest for subworkflow 'bam_rseqc' diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index a9844d80a9..fd0a2cd8f3 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-3054113778-matrix { + .terminal-3261404162-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3054113778-title { + .terminal-3261404162-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3054113778-r1 { fill: #c5c8c6 } -.terminal-3054113778-r2 { fill: #98a84b } -.terminal-3054113778-r3 { fill: #9a9b99 } -.terminal-3054113778-r4 { fill: #608ab1 } -.terminal-3054113778-r5 { fill: #d0b344 } -.terminal-3054113778-r6 { fill: #cc555a;font-weight: bold } + .terminal-3261404162-r1 { fill: #c5c8c6 } +.terminal-3261404162-r2 { fill: #98a84b } +.terminal-3261404162-r3 { fill: #9a9b99 } +.terminal-3261404162-r4 { fill: #608ab1 } +.terminal-3261404162-r5 { fill: #d0b344 } +.terminal-3261404162-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -ERROR   'rseqc/junctionannotation' + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +ERROR   'rseqc/junctionsaturation' diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 7305dda430..cc7f415405 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + - + - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master' -INFO     Deleting all files in 'TEMPLATE' branch                                                     -INFO     Making a new template pipeline using pipeline variables                                     + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master' +INFO     Deleting all files in 'TEMPLATE' branch                                                     +INFO     Making a new template pipeline using pipeline variables                                     +INFO     Committed changes to 'TEMPLATE' branch                                                      +INFO     Checking out original branch: 'master' +INFO     Now try to merge the updates in to your pipeline:                                           +           cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +           git merge TEMPLATE                                                                        From 14aac7a1865b8ec076339af0bce7d6f7bde6f518 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Jun 2023 10:36:33 +0200 Subject: [PATCH 231/249] fix download command for rich-codex in the readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce7c8a5690..8efcff18da 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,7 @@ Each option has a flag, if all are supplied then it will run without any user in working_dir: tmp --> -![`nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none`](docs/images/nf-core-download.svg) +![`nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -s none -d`](docs/images/nf-core-download.svg) Once downloaded, you will see something like the following file structure for the downloaded pipeline: From 96dfa188d15d14bf9e3b04bedd099bfabc6a101c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Jun 2023 10:39:45 +0200 Subject: [PATCH 232/249] update download path for nf-core schema validate image --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8efcff18da..b215c01efb 100644 --- a/README.md +++ b/README.md @@ -571,7 +571,7 @@ timeout: 10 after_command: rm nf-params.json --> -![`nf-core schema validate nf-core-rnaseq/workflow nf-params.json`](docs/images/nf-core-schema-validate.svg) +![`nf-core schema validate nf-core-rnaseq/3_8 nf-params.json`](docs/images/nf-core-schema-validate.svg) The `pipeline` option can be a directory containing a pipeline, a path to a schema file or the name of an nf-core pipeline (which will be downloaded using `nextflow pull`). From f6cf958b7641fb7debcdd9cdd12fa09cf1081ecf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 30 Jun 2023 08:47:40 +0000 Subject: [PATCH 233/249] Generate new screengrabs with rich-codex --- docs/images/nf-core-download-tree.svg | 152 ++- docs/images/nf-core-download.svg | 101 +- docs/images/nf-core-licences.svg | 135 +-- docs/images/nf-core-list.svg | 108 +-- docs/images/nf-core-modules-create-test.svg | 98 +- docs/images/nf-core-schema-validate.svg | 900 +----------------- .../nf-core-subworkflows-create-test.svg | 105 +- docs/images/nf-core-subworkflows-create.svg | 116 +-- docs/images/nf-core-subworkflows-remove.svg | 103 +- docs/images/nf-core-subworkflows-test.svg | 88 +- docs/images/nf-core-subworkflows-update.svg | 72 +- docs/images/nf-core-sync.svg | 108 +-- 12 files changed, 606 insertions(+), 1480 deletions(-) diff --git a/docs/images/nf-core-download-tree.svg b/docs/images/nf-core-download-tree.svg index 670c1330fe..fc9585c8c9 100644 --- a/docs/images/nf-core-download-tree.svg +++ b/docs/images/nf-core-download-tree.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ tree -L 2 nf-core-rnaseq/ -nf-core-rnaseq/  [error opening dir] - -0 directories, 0 files + + $ tree -L 2 nf-core-rnaseq/ +nf-core-rnaseq/ +├── 3_8 +│   ├── CHANGELOG.md +│   ├── CITATIONS.md +│   ├── CODE_OF_CONDUCT.md +│   ├── LICENSE +│   ├── README.md +│   ├── assets +│   ├── bin +│   ├── conf +│   ├── docs +│   ├── lib +│   ├── main.nf +│   ├── modules +│   ├── modules.json +│   ├── nextflow.config +│   ├── nextflow_schema.json +│   ├── subworkflows +│   ├── tower.yml +│   └── workflows +└── configs +    ├── CITATION.cff +    ├── LICENSE +    ├── README.md +    ├── bin +    ├── conf +    ├── configtest.nf +    ├── docs +    ├── nextflow.config +    ├── nfcore_custom.config +    └── pipeline + +14 directories, 16 files diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 814b149d02..69b0b5b29b 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,89 +19,82 @@ font-weight: 700; } - .terminal-1804501394-matrix { + .terminal-349809940-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1804501394-title { + .terminal-349809940-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1804501394-r1 { fill: #c5c8c6 } -.terminal-1804501394-r2 { fill: #98a84b } -.terminal-1804501394-r3 { fill: #9a9b99 } -.terminal-1804501394-r4 { fill: #608ab1 } -.terminal-1804501394-r5 { fill: #d0b344 } -.terminal-1804501394-r6 { fill: #68a0b3;font-weight: bold } -.terminal-1804501394-r7 { fill: #d0b344;font-weight: bold } -.terminal-1804501394-r8 { fill: #868887 } -.terminal-1804501394-r9 { fill: #4a637a } -.terminal-1804501394-r10 { fill: #4a637a;font-weight: bold } -.terminal-1804501394-r11 { fill: #cc555a } -.terminal-1804501394-r12 { fill: #98a84b;font-weight: bold } + .terminal-349809940-r1 { fill: #c5c8c6 } +.terminal-349809940-r2 { fill: #98a84b } +.terminal-349809940-r3 { fill: #9a9b99 } +.terminal-349809940-r4 { fill: #608ab1 } +.terminal-349809940-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -113,28 +106,28 @@ - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -Usage: nf-core download [OPTIONS] <pipeline name> - -Try 'nf-core download -h' for help. -╭─ Error ──────────────────────────────────────────────────────────────────────────────────────────╮ -│ No such option: -c│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -s none -d + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO     Saving 'nf-core/rnaseq' +          Pipeline revision: '3.8' +          Use containers: 'none' +          Container library: 'quay.io' +          Output directory: 'nf-core-rnaseq' +          Include default institutional configuration: 'True' +INFO     Downloading centralised configs from GitHub                                                 +INFO     Downloading workflow files from GitHub                                                      diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index 9a72e4beda..2c436150e9 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    -INFO     Warning: This tool only prints licence information for the software tools packaged using    -         conda.                                                                                      -INFO     The pipeline may use other software and dependencies not described here.                    -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index cd47de283f..001d4b8ece 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-457079933-matrix { + .terminal-3824450644-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-457079933-title { + .terminal-3824450644-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-457079933-r1 { fill: #c5c8c6 } -.terminal-457079933-r2 { fill: #98a84b } -.terminal-457079933-r3 { fill: #9a9b99 } -.terminal-457079933-r4 { fill: #608ab1 } -.terminal-457079933-r5 { fill: #d0b344 } -.terminal-457079933-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-457079933-r7 { fill: #868887 } -.terminal-457079933-r8 { fill: #868887;font-style: italic; } + .terminal-3824450644-r1 { fill: #c5c8c6 } +.terminal-3824450644-r2 { fill: #98a84b } +.terminal-3824450644-r3 { fill: #9a9b99 } +.terminal-3824450644-r4 { fill: #608ab1 } +.terminal-3824450644-r5 { fill: #d0b344 } +.terminal-3824450644-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3824450644-r7 { fill: #868887 } +.terminal-3824450644-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ funcscan             │    35 │          1.1.2 │  an hour ago │           - │ -                   │ -│ ampliseq             │   118 │          2.6.1 │   2 days ago │           - │ -                   │ -│ circdna              │    16 │          1.0.4 │   2 days ago │           - │ -                   │ -│ marsseq              │     3 │          1.0.1 │   4 days ago │           - │ -                   │ -│ nanostring           │     4 │          1.1.1 │   6 days ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ funcscan             │    35 │          1.1.2 │  2 hours ago │           - │ -                   │ +│ ampliseq             │   118 │          2.6.1 │   2 days ago │           - │ -                   │ +│ circdna              │    16 │          1.0.4 │   3 days ago │           - │ -                   │ +│ marsseq              │     3 │          1.0.1 │   4 days ago │           - │ -                   │ +│ nanostring           │     4 │          1.1.1 │   6 days ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 6ba5712594..320dde2c21 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,84 +19,84 @@ font-weight: 700; } - .terminal-3820264502-matrix { + .terminal-4099513362-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3820264502-title { + .terminal-4099513362-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3820264502-r1 { fill: #c5c8c6 } -.terminal-3820264502-r2 { fill: #98a84b } -.terminal-3820264502-r3 { fill: #9a9b99 } -.terminal-3820264502-r4 { fill: #608ab1 } -.terminal-3820264502-r5 { fill: #d0b344 } -.terminal-3820264502-r6 { fill: #ff2c7a } -.terminal-3820264502-r7 { fill: #98729f } + .terminal-4099513362-r1 { fill: #c5c8c6 } +.terminal-4099513362-r2 { fill: #98a84b } +.terminal-4099513362-r3 { fill: #9a9b99 } +.terminal-4099513362-r4 { fill: #608ab1 } +.terminal-4099513362-r5 { fill: #d0b344 } +.terminal-4099513362-r6 { fill: #ff2c7a } +.terminal-4099513362-r7 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108,28 +108,28 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc' -INFO     Running 'fastqc' test with command:                                                         -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  -./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmpk7h2oenp -work-dir /tmp/tmpb21y__ar + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmpdheu0fmi -work-dir /tmp/tmprhu112t0 diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 14ea6ff059..b7fe83a26e 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - -╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ -│/opt/hostedtoolcache/Python/3.11.4/x64/bin/nf-core:8 in <module>│ -││ -│5 fromnf_core.__main__import run_nf_core                                                     │ -│6 if__name__ == '__main__':                                                                   │ -│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ -│❱ 8 │   sys.exit(run_nf_core())                                                                  │ -│9 │ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/__main__.py:110 in   │ -│run_nf_core│ -││ -│ 107 │   │   │   log.debug(f"Could not check latest version: {e}")                             │ -│ 108 │   │   stderr.print("\n")                                                                │ -│ 109 │   # Launch the click cli│ -│❱  110 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ -│ 111 │ -│ 112 │ -│ 113 @click.group(context_settings=dict(help_option_names=["-h", "--help"]))                   │ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1130 in        │ -│__call__│ -││ -│1127 │   │ -│1128 │   def__call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any:                           │ -│1129 │   │   """Alias for :meth:`main`."""│ -│❱ 1130 │   │   returnself.main(*args, **kwargs)                                                 │ -│1131 │ -│1132 │ -│1133 classCommand(BaseCommand):                                                               │ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/rich_click/rich_group.py:21│ -│ in main│ -││ -│18 │   │ -│19 │   defmain(self, *args, standalone_mode: bool = True, **kwargs):                          │ -│20 │   │   try:                                                                                │ -│❱ 21 │   │   │   rv = super().main(*args, standalone_mode=False, **kwargs)                       │ -│22 │   │   │   ifnot standalone_mode:                                                         │ -│23 │   │   │   │   return rv                                                                   │ -│24 │   │   except click.ClickException as e:                                                   │ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1055 in main│ -││ -│1052 │   │   try:                                                                              │ -│1053 │   │   │   try:                                                                          │ -│1054 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                  │ -│❱ 1055 │   │   │   │   │   rv = self.invoke(ctx)                                                 │ -│1056 │   │   │   │   │   ifnot standalone_mode:                                               │ -│1057 │   │   │   │   │   │   return rv                                                         │ -│1058 │   │   │   │   │   # it's not safe to `ctx.exit(rv)` here!│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ -││ -│1654 │   │   │   │   super().invoke(ctx)                                                       │ -│1655 │   │   │   │   sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)                    │ -│1656 │   │   │   │   with sub_ctx:                                                             │ -│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ -│1658 │   │   │ -│1659 │   │   # In chain mode we create the contexts step by step, but after the│ -│1660 │   │   # base command has been invoked.  Because at that point we do not│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ -││ -│1654 │   │   │   │   super().invoke(ctx)                                                       │ -│1655 │   │   │   │   sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)                    │ -│1656 │   │   │   │   with sub_ctx:                                                             │ -│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ -│1658 │   │   │ -│1659 │   │   # In chain mode we create the contexts step by step, but after the│ -│1660 │   │   # base command has been invoked.  Because at that point we do not│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:1404 in invoke│ -││ -│1401 │   │   │   echo(style(message, fg="red"), err=True)                                      │ -│1402 │   │   │ -│1403 │   │   ifself.callback isnotNone:                                                     │ -│❱ 1404 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ -│1405 │   │ -│1406 │   defshell_complete(self, ctx: Context, incomplete: str) -> t.List["CompletionItem"]:  │ -│1407 │   │   """Return a list of completions for the incomplete value. Looks│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/click/core.py:760 in invoke│ -││ -│ 757 │   │   │ -│ 758 │   │   with augment_usage_errors(__self):                                                │ -│ 759 │   │   │   with ctx:                                                                     │ -│❱  760 │   │   │   │   return __callback(*args, **kwargs)                                        │ -│ 761 │   │ -│ 762 │   defforward(                                                                          │ -│ 763 │   │   __self, __cmd: "Command", *args: t.Any, **kwargs: t.Any  # noqa: B902│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/__main__.py:1369 in  │ -│validate│ -││ -│1366 │   │ -│1367 │   schema_obj = PipelineSchema()                                                         │ -│1368 │   try:                                                                                  │ -│❱ 1369 │   │   schema_obj.get_schema_path(pipeline)                                              │ -│1370 │   │   # Load and check schema│ -│1371 │   │   schema_obj.load_lint_schema()                                                     │ -│1372 │   exceptAssertionErroras e:                                                           │ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/schema.py:66 in      │ -│get_schema_path│ -││ -│ 63 │   │   │ -│ 64 │   │   # Path does not exist - assume a name of a remote workflow│ -│ 65 │   │   elifnot local_only:                                                               │ -│❱  66 │   │   │   self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision)         │ -│ 67 │   │   │   self.schema_filename = os.path.join(self.pipeline_dir, "nextflow_schema.json│ -│ 68 │   │   │ -│ 69 │   │   # Only looking for local paths, overwrite with None to be safe│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/list.py:52 in        │ -│get_local_wf│ -││ -│ 49 │   │   workflow = f"nf-core/{workflow}"│ -│ 50 │   │ -│ 51 │   wfs = Workflows()                                                                      │ -│❱  52 │   wfs.get_local_nf_workflows()                                                           │ -│ 53 │   for wf in wfs.local_workflows:                                                         │ -│ 54 │   │   if workflow == wf.full_name:                                                       │ -│ 55 │   │   │   if revision isNoneor revision == wf.commit_sha or revision == wf.branch or│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/list.py:141 in       │ -│get_local_nf_workflows│ -││ -│138 │   │   # Find additional information about each workflow by checking its git history│ -│139 │   │   log.debug(f"Fetching extra info about {len(self.local_workflows)} local workflow│ -│140 │   │   for wf inself.local_workflows:                                                    │ -│❱ 141 │   │   │   wf.get_local_nf_workflow_details()                                             │ -│142 │   │ -│143 │   defcompare_remote_local(self):                                                        │ -│144 │   │   """Matches local to remote workflows.│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/nf_core/list.py:356 in       │ -│get_local_nf_workflow_details│ -││ -│353 │   │   │   log.debug(f"Pulling git info from {self.local_path}")                          │ -│354 │   │   │   try:                                                                           │ -│355 │   │   │   │   repo = git.Repo(self.local_path)                                           │ -│❱ 356 │   │   │   │   self.commit_sha = str(repo.head.commit.hexsha)                             │ -│357 │   │   │   │   self.remote_url = str(repo.remotes.origin.url)                             │ -│358 │   │   │   │   self.last_pull = os.stat(os.path.join(self.local_path, ".git", "FETCH_HE│ -│359 │   │   │   │   self.last_pull_date = datetime.fromtimestamp(self.last_pull).strftime("%│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:226 in  │ -│_get_commit│ -││ -│223 │   │   :return:│ -│224 │   │   │   Commit object we point to, works for detached and non-detached│ -│225 │   │   │   SymbolicReferences. The symbolic reference will be dereferenced recursively.│ -│❱ 226 │   │   obj = self._get_object()                                                           │ -│227 │   │   if obj.type == "tag":                                                              │ -│228 │   │   │   obj = obj.object                                                               │ -│229 │   │   # END dereference tag│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:219 in  │ -│_get_object│ -││ -│216 │   │   │   always point to the actual object as it gets re-created on each query"""│ -│217 │   │   # have to be dynamic here as we may be a tag which can point to anything│ -│218 │   │   # Our path will be resolved to the hexsha which will be used accordingly│ -│❱ 219 │   │   return Object.new_from_sha(self.repo, hex_to_bin(self.dereference_recursive(self│ -│220 │   │ -│221 │   def_get_commit(self) -> "Commit":                                                     │ -│222 │   │   """│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:159 in  │ -│dereference_recursive│ -││ -│156 │   │   :param repo: the repository containing the reference at ref_path"""│ -│157 │   │   │ -│158 │   │   whileTrue:                                                                        │ -│❱ 159 │   │   │   hexsha, ref_path = cls._get_ref_info(repo, ref_path)                           │ -│160 │   │   │   if hexsha isnotNone:                                                         │ -│161 │   │   │   │   return hexsha                                                              │ -│162 │   │   # END recursive dereferencing│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:210 in  │ -│_get_ref_info│ -││ -│207 │   │   """Return: (str(sha), str(target_ref_path)) if available, the sha the file at│ -│208 │   │   rela_path points to, or None. target_ref_path is the reference we│ -│209 │   │   point to, or None"""│ -│❱ 210 │   │   returncls._get_ref_info_helper(repo, ref_path)                                    │ -│211 │   │ -│212 │   def_get_object(self) -> Commit_ish:                                                   │ -│213 │   │   """│ -││ -│/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/site-packages/git/refs/symbolic.py:193 in  │ -│_get_ref_info_helper│ -││ -│190 │   │   │   # END for each packed ref│ -│191 │   │   # END handle packed refs│ -│192 │   │   if tokens isNone:                                                                 │ -│❱ 193 │   │   │   raiseValueError("Reference at %r does not exist" % ref_path)                  │ -│194 │   │   │ -│195 │   │   # is it a reference ?│ -│196 │   │   if tokens[0] == "ref:":                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -ValueError: Reference at 'refs/heads/master' does not exist + + $ nf-core schema validate nf-core-rnaseq/3_8 nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 93 params) +INFO    [✓] Input parameters look valid diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index 07bc2df151..2c6f7f2018 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Looking for test workflow entry points:                                                     -'tests/subworkflows/nf-core/bam_stats_samtools/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_bam_stats_samtools' -INFO     Running 'bam_stats_samtools' test with command:                                             -nextflow run ./tests/subworkflows/nf-core/bam_stats_samtools -entry test_bam_stats_samtools --c ./tests/config/nextflow.config --outdir /tmp/tmp05wh1761 -work-dir /tmp/tmpwg_j73w9 + + + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index df2311a2fd..6a302200ac 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Created / edited following files:                                                           -           ./subworkflows/nf-core/bam_stats_samtools/main.nf -           ./subworkflows/nf-core/bam_stats_samtools/meta.yml -           ./tests/subworkflows/nf-core/bam_stats_samtools/main.nf -           ./tests/subworkflows/nf-core/bam_stats_samtools/test.yml -           ./tests/subworkflows/nf-core/bam_stats_samtools/nextflow.config -           ./tests/config/pytest_modules.yml + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 040c8d999b..7b71d8010e 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Removed files for 'rseqc/bamstat' and it's dependencies 'rseqc/bamstat'.                    -INFO     Removed files for 'rseqc/inferexperiment' and it's dependencies 'rseqc/inferexperiment'.    -INFO     Removed files for 'rseqc/innerdistance' and it's dependencies 'rseqc/innerdistance'.        -INFO     Removed files for 'rseqc/junctionannotation' and it's dependencies                          -'rseqc/junctionannotation'.                                                                 -INFO     Removed files for 'bam_rseqc' and it's dependencies 'bam_rseqc, rseqc_bamstat,  -rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation'.                      + + + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index 459502886a..6319ce11c4 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - - - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -──────────────────────────────────────────── bam_rseqc ───────────────────────────────────────────── -WARNING  You have uncommitted changes. Make sure to commit last changes before running the tests.    -INFO     Running pytest for subworkflow 'bam_rseqc' + + + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index fd0a2cd8f3..3b21d86571 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-3261404162-matrix { + .terminal-3965598301-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3261404162-title { + .terminal-3965598301-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3261404162-r1 { fill: #c5c8c6 } -.terminal-3261404162-r2 { fill: #98a84b } -.terminal-3261404162-r3 { fill: #9a9b99 } -.terminal-3261404162-r4 { fill: #608ab1 } -.terminal-3261404162-r5 { fill: #d0b344 } -.terminal-3261404162-r6 { fill: #cc555a;font-weight: bold } + .terminal-3965598301-r1 { fill: #c5c8c6 } +.terminal-3965598301-r2 { fill: #98a84b } +.terminal-3965598301-r3 { fill: #9a9b99 } +.terminal-3965598301-r4 { fill: #608ab1 } +.terminal-3965598301-r5 { fill: #d0b344 } +.terminal-3965598301-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -ERROR   'rseqc/junctionsaturation' + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index cc7f415405..d8c77548b5 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - + - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master' -INFO     Deleting all files in 'TEMPLATE' branch                                                     -INFO     Making a new template pipeline using pipeline variables                                     -INFO     Committed changes to 'TEMPLATE' branch                                                      -INFO     Checking out original branch: 'master' -INFO     Now try to merge the updates in to your pipeline:                                           -           cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -           git merge TEMPLATE                                                                        + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master' +INFO     Deleting all files in 'TEMPLATE' branch                                                     +INFO     Making a new template pipeline using pipeline variables                                     From fe8365597422665ee5d83986bd6ed3710a1856d1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Jun 2023 11:04:37 +0200 Subject: [PATCH 234/249] rich-codex remove modules repo folder before clone --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b215c01efb..c68f51086f 100644 --- a/README.md +++ b/README.md @@ -908,7 +908,7 @@ The `nf-core modules create` command will prompt you with the relevant questions @@ -1181,7 +1181,7 @@ The `nf-core subworkflows create` command will prompt you with the relevant ques From 185ecf0a83a76532233cc7c7c235dea31da1628b Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 30 Jun 2023 11:23:17 +0200 Subject: [PATCH 235/249] Update nextflow.config --- nf_core/pipeline-template/nextflow.config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 66e547489a..ae62a6ecac 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -179,12 +179,13 @@ profiles { test_full { includeConfig 'conf/test_full.config' } } -// Set default registry for Docker and Podman independent of -profile -// Will not be used unless Docker / Podman are enabled +// Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Singularity are enabled // Set to your registry if you have a mirror of containers -singularity.registry = 'quay.io' +apptainer.registry = 'quay.io' docker.registry = 'quay.io' podman.registry = 'quay.io' +singularity.registry = 'quay.io' // Nextflow plugins plugins { From 9d3971627e715d53fae98b330d58c06f4c690dd1 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 30 Jun 2023 11:26:12 +0200 Subject: [PATCH 236/249] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47fa728b37..8e4e15c568 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306)) - Add ability to specify custom registry for linting modules, defaults to quay.io ([#2313](https://github.com/nf-core/tools/pull/2313)) - Add `singularity.registry = 'quay.io'` in pipeline template ([#2305](https://github.com/nf-core/tools/pull/2305)) +- Add `apptainer.registry = 'quay.io'` in pipeline template ([#2305](https://github.com/nf-core/tools/pull/2352)) - Bump minimum required NF version in pipeline template from `22.10.1` -> `23.04.0` ([#2305](https://github.com/nf-core/tools/pull/2305)) - Add ability to interpret `docker.registry` from `nextflow.config` file. If not found defaults to quay.io. ([#2318](https://github.com/nf-core/tools/pull/2318)) - Add functions to dynamically include pipeline tool citations in MultiQC methods description section for better reporting. ([#2326](https://github.com/nf-core/tools/pull/2326)) From de0c24723e6bc81b4320fd421f1ad2c0df88c903 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 30 Jun 2023 11:34:52 +0200 Subject: [PATCH 237/249] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4e15c568..77196dd428 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ - Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306)) - Add ability to specify custom registry for linting modules, defaults to quay.io ([#2313](https://github.com/nf-core/tools/pull/2313)) - Add `singularity.registry = 'quay.io'` in pipeline template ([#2305](https://github.com/nf-core/tools/pull/2305)) -- Add `apptainer.registry = 'quay.io'` in pipeline template ([#2305](https://github.com/nf-core/tools/pull/2352)) +- Add `apptainer.registry = 'quay.io'` in pipeline template ([#2352](https://github.com/nf-core/tools/pull/2352)) - Bump minimum required NF version in pipeline template from `22.10.1` -> `23.04.0` ([#2305](https://github.com/nf-core/tools/pull/2305)) - Add ability to interpret `docker.registry` from `nextflow.config` file. If not found defaults to quay.io. ([#2318](https://github.com/nf-core/tools/pull/2318)) - Add functions to dynamically include pipeline tool citations in MultiQC methods description section for better reporting. ([#2326](https://github.com/nf-core/tools/pull/2326)) From d2b8a45d6387a99aa4abc266cfcb7b8bbb9b17cf Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 30 Jun 2023 12:00:35 +0200 Subject: [PATCH 238/249] Add apptainer keyword to the list of false positive matches. Bare minimum to possibly support a future apptainer.registry notation. --- nf_core/download.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index c5d0e2de60..f049b41ab5 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -733,8 +733,7 @@ def find_container_images(self, workflow_directory): re.DOTALL is used to account for the string to be spread out across multiple lines. """ container_regex = re.compile( - r"container\s+[\s{}=$]*(?P[\'\"])(?P(?:.(?!\1))*.?)\1[\s}]*", - re.DOTALL, + r"container\s+[\s{}=$]*(?P[\'\"])(?P(?:.(?!\1))*.?)\1[\s}]*", re.DOTALL ) local_module_findings = re.findall(container_regex, search_space) @@ -850,7 +849,7 @@ def rectify_raw_container_matches(self, raw_findings): for _, capture in container_value_defs: # common false positive(s) - if capture in ["singularity"]: + if capture in ["singularity", "apptainer"]: continue # Look for a http download URL. From b2e906dd5f18d874a21cd50f16e5fb3e9b8ac74f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 30 Jun 2023 10:27:10 +0000 Subject: [PATCH 239/249] Generate new screengrabs with rich-codex --- docs/images/nf-core-licences.svg | 135 +++++++++++----- docs/images/nf-core-list.svg | 108 ++++++------- docs/images/nf-core-modules-bump-version.svg | 106 ++++++------ docs/images/nf-core-modules-create-test.svg | 122 +++++--------- docs/images/nf-core-modules-create.svg | 153 ++++++------------ docs/images/nf-core-modules-lint.svg | 100 +++++------- docs/images/nf-core-modules-test.svg | 85 +++++----- .../nf-core-subworkflows-create-test.svg | 99 ++++++------ docs/images/nf-core-subworkflows-create.svg | 97 +++++------ docs/images/nf-core-subworkflows-remove.svg | 103 +++++++----- docs/images/nf-core-subworkflows-test.svg | 88 +++++----- docs/images/nf-core-subworkflows-update.svg | 72 ++++----- 12 files changed, 616 insertions(+), 652 deletions(-) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index 2c436150e9..9a72e4beda 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    +INFO     Warning: This tool only prints licence information for the software tools packaged using    +         conda.                                                                                      +INFO     The pipeline may use other software and dependencies not described here.                    +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 001d4b8ece..4f7a689cd5 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-3824450644-matrix { + .terminal-4076108886-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3824450644-title { + .terminal-4076108886-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3824450644-r1 { fill: #c5c8c6 } -.terminal-3824450644-r2 { fill: #98a84b } -.terminal-3824450644-r3 { fill: #9a9b99 } -.terminal-3824450644-r4 { fill: #608ab1 } -.terminal-3824450644-r5 { fill: #d0b344 } -.terminal-3824450644-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3824450644-r7 { fill: #868887 } -.terminal-3824450644-r8 { fill: #868887;font-style: italic; } + .terminal-4076108886-r1 { fill: #c5c8c6 } +.terminal-4076108886-r2 { fill: #98a84b } +.terminal-4076108886-r3 { fill: #9a9b99 } +.terminal-4076108886-r4 { fill: #608ab1 } +.terminal-4076108886-r5 { fill: #d0b344 } +.terminal-4076108886-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4076108886-r7 { fill: #868887 } +.terminal-4076108886-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ funcscan             │    35 │          1.1.2 │  2 hours ago │           - │ -                   │ -│ ampliseq             │   118 │          2.6.1 │   2 days ago │           - │ -                   │ -│ circdna              │    16 │          1.0.4 │   3 days ago │           - │ -                   │ -│ marsseq              │     3 │          1.0.1 │   4 days ago │           - │ -                   │ -│ nanostring           │     4 │          1.1.1 │   6 days ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ funcscan             │    35 │          1.1.2 │  4 hours ago │           - │ -                   │ +│ ampliseq             │   118 │          2.6.1 │   2 days ago │           - │ -                   │ +│ circdna              │    16 │          1.0.4 │   3 days ago │           - │ -                   │ +│ marsseq              │     3 │          1.0.1 │   4 days ago │           - │ -                   │ +│ nanostring           │     4 │          1.1.1 │   6 days ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 3bafb91264..755e5651e1 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-1316878266-matrix { + .terminal-1176957473-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1316878266-title { + .terminal-1176957473-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1316878266-r1 { fill: #c5c8c6 } -.terminal-1316878266-r2 { fill: #98a84b } -.terminal-1316878266-r3 { fill: #9a9b99 } -.terminal-1316878266-r4 { fill: #608ab1 } -.terminal-1316878266-r5 { fill: #d0b344 } -.terminal-1316878266-r6 { fill: #98a84b;font-weight: bold } -.terminal-1316878266-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-1176957473-r1 { fill: #c5c8c6 } +.terminal-1176957473-r2 { fill: #98a84b } +.terminal-1176957473-r3 { fill: #9a9b99 } +.terminal-1176957473-r4 { fill: #608ab1 } +.terminal-1176957473-r5 { fill: #d0b344 } +.terminal-1176957473-r6 { fill: #d0b344;font-weight: bold } +.terminal-1176957473-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module updated│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module updated:  0.11.9 --> 0.12.1                    │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 320dde2c21..32997fd386 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc' -INFO     Running 'fastqc' test with command:                                                         -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  -./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmpdheu0fmi -work-dir /tmp/tmprhu112t0 + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 70a03b29a7..9490500715 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' -INFO     Using Docker container: 'biocontainers/fastqc:0.12.1--hdfd78af_0' -INFO     Using Singularity container:                                                                -'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' -INFO     Created / edited following files:                                                           -           ./modules/nf-core/fastqc/main.nf -           ./modules/nf-core/fastqc/meta.yml -           ./tests/modules/nf-core/fastqc/main.nf -           ./tests/modules/nf-core/fastqc/test.yml -           ./tests/modules/nf-core/fastqc/nextflow.config -           ./tests/config/pytest_modules.yml + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low  +--meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 08d62c2f3c..efbb11ffe5 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - + + - + - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 3c6eeaca9e..5b235b45a5 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -WARNING  You have uncommitted changes. Make sure to commit last changes before running the tests.    -INFO     Running pytest for module 'samtools/view' + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view' diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index 2c6f7f2018..3be8678215 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - + + - + - - - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index 6a302200ac..3454ab8a62 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - + + - + - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 7b71d8010e..040c8d999b 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Removed files for 'rseqc/bamstat' and it's dependencies 'rseqc/bamstat'.                    +INFO     Removed files for 'rseqc/inferexperiment' and it's dependencies 'rseqc/inferexperiment'.    +INFO     Removed files for 'rseqc/innerdistance' and it's dependencies 'rseqc/innerdistance'.        +INFO     Removed files for 'rseqc/junctionannotation' and it's dependencies                          +'rseqc/junctionannotation'.                                                                 +INFO     Removed files for 'bam_rseqc' and it's dependencies 'bam_rseqc, rseqc_bamstat,  +rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation'.                      diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index 6319ce11c4..459502886a 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - - - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +──────────────────────────────────────────── bam_rseqc ───────────────────────────────────────────── +WARNING  You have uncommitted changes. Make sure to commit last changes before running the tests.    +INFO     Running pytest for subworkflow 'bam_rseqc' diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index 3b21d86571..fd0a2cd8f3 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-3965598301-matrix { + .terminal-3261404162-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3965598301-title { + .terminal-3261404162-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3965598301-r1 { fill: #c5c8c6 } -.terminal-3965598301-r2 { fill: #98a84b } -.terminal-3965598301-r3 { fill: #9a9b99 } -.terminal-3965598301-r4 { fill: #608ab1 } -.terminal-3965598301-r5 { fill: #d0b344 } -.terminal-3965598301-r6 { fill: #cc555a;font-weight: bold } + .terminal-3261404162-r1 { fill: #c5c8c6 } +.terminal-3261404162-r2 { fill: #98a84b } +.terminal-3261404162-r3 { fill: #9a9b99 } +.terminal-3261404162-r4 { fill: #608ab1 } +.terminal-3261404162-r5 { fill: #d0b344 } +.terminal-3261404162-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +ERROR   'rseqc/junctionsaturation' From 948de3fd1a66226f7fd8528c127c1297c66be797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 30 Jun 2023 12:59:35 +0200 Subject: [PATCH 240/249] Revert "rich-codex remove modules repo folder before clone" --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c68f51086f..b215c01efb 100644 --- a/README.md +++ b/README.md @@ -908,7 +908,7 @@ The `nf-core modules create` command will prompt you with the relevant questions @@ -1181,7 +1181,7 @@ The `nf-core subworkflows create` command will prompt you with the relevant ques From 4dee2c58be72aae32f8e47afee2a14b5b42b6f5a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Jun 2023 13:10:57 +0200 Subject: [PATCH 241/249] don't clone modules repo again --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index ce7c8a5690..df531925ca 100644 --- a/README.md +++ b/README.md @@ -1180,8 +1180,7 @@ It will start in the current working directory, or whatever is specified with `- The `nf-core subworkflows create` command will prompt you with the relevant questions in order to create all of the necessary subworkflow files. From e101f14d81a29405427954992bc9e34b7621c2c5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Jun 2023 13:16:27 +0200 Subject: [PATCH 242/249] add timeout when cloning modules repo --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index df531925ca..cc7fdefafe 100644 --- a/README.md +++ b/README.md @@ -908,6 +908,7 @@ The `nf-core modules create` command will prompt you with the relevant questions From 6a96cece1d2eb11e6a6d60644ac330ae5d3462f4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 30 Jun 2023 11:22:01 +0000 Subject: [PATCH 243/249] Generate new screengrabs with rich-codex --- docs/images/nf-core-list.svg | 108 ++++++------- docs/images/nf-core-modules-bump-version.svg | 106 ++++++------ docs/images/nf-core-modules-create-test.svg | 122 +++++++++----- docs/images/nf-core-modules-create.svg | 153 ++++++++++++------ docs/images/nf-core-modules-lint.svg | 100 +++++++----- docs/images/nf-core-modules-test.svg | 85 +++++----- .../nf-core-subworkflows-create-test.svg | 126 ++++++++++----- docs/images/nf-core-subworkflows-create.svg | 112 +++++++------ 8 files changed, 541 insertions(+), 371 deletions(-) diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 4f7a689cd5..92324289f4 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-4076108886-matrix { + .terminal-4201938007-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4076108886-title { + .terminal-4201938007-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4076108886-r1 { fill: #c5c8c6 } -.terminal-4076108886-r2 { fill: #98a84b } -.terminal-4076108886-r3 { fill: #9a9b99 } -.terminal-4076108886-r4 { fill: #608ab1 } -.terminal-4076108886-r5 { fill: #d0b344 } -.terminal-4076108886-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-4076108886-r7 { fill: #868887 } -.terminal-4076108886-r8 { fill: #868887;font-style: italic; } + .terminal-4201938007-r1 { fill: #c5c8c6 } +.terminal-4201938007-r2 { fill: #98a84b } +.terminal-4201938007-r3 { fill: #9a9b99 } +.terminal-4201938007-r4 { fill: #608ab1 } +.terminal-4201938007-r5 { fill: #d0b344 } +.terminal-4201938007-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4201938007-r7 { fill: #868887 } +.terminal-4201938007-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ funcscan             │    35 │          1.1.2 │  4 hours ago │           - │ -                   │ -│ ampliseq             │   118 │          2.6.1 │   2 days ago │           - │ -                   │ -│ circdna              │    16 │          1.0.4 │   3 days ago │           - │ -                   │ -│ marsseq              │     3 │          1.0.1 │   4 days ago │           - │ -                   │ -│ nanostring           │     4 │          1.1.1 │   6 days ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ funcscan             │    35 │          1.1.2 │  5 hours ago │           - │ -                   │ +│ ampliseq             │   118 │          2.6.1 │   2 days ago │           - │ -                   │ +│ circdna              │    16 │          1.0.4 │   3 days ago │           - │ -                   │ +│ marsseq              │     3 │          1.0.1 │   4 days ago │           - │ -                   │ +│ nanostring           │     4 │          1.1.1 │   6 days ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 755e5651e1..3bafb91264 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-1176957473-matrix { + .terminal-1316878266-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1176957473-title { + .terminal-1316878266-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1176957473-r1 { fill: #c5c8c6 } -.terminal-1176957473-r2 { fill: #98a84b } -.terminal-1176957473-r3 { fill: #9a9b99 } -.terminal-1176957473-r4 { fill: #608ab1 } -.terminal-1176957473-r5 { fill: #d0b344 } -.terminal-1176957473-r6 { fill: #d0b344;font-weight: bold } -.terminal-1176957473-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-1316878266-r1 { fill: #c5c8c6 } +.terminal-1316878266-r2 { fill: #98a84b } +.terminal-1316878266-r3 { fill: #9a9b99 } +.terminal-1316878266-r4 { fill: #608ab1 } +.terminal-1316878266-r5 { fill: #d0b344 } +.terminal-1316878266-r6 { fill: #98a84b;font-weight: bold } +.terminal-1316878266-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module updated│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module updated:  0.11.9 --> 0.12.1                    │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 32997fd386..40edde66fe 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmp1dv_lyip -work-dir /tmp/tmporw8oqxx diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 9490500715..70a03b29a7 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low  ---meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' +INFO     Using Docker container: 'biocontainers/fastqc:0.12.1--hdfd78af_0' +INFO     Using Singularity container:                                                                +'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' +INFO     Created / edited following files:                                                           +           ./modules/nf-core/fastqc/main.nf +           ./modules/nf-core/fastqc/meta.yml +           ./tests/modules/nf-core/fastqc/main.nf +           ./tests/modules/nf-core/fastqc/test.yml +           ./tests/modules/nf-core/fastqc/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index efbb11ffe5..08d62c2f3c 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + - + - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 5b235b45a5..3c6eeaca9e 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view' + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +WARNING  You have uncommitted changes. Make sure to commit last changes before running the tests.    +INFO     Running pytest for module 'samtools/view' diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index 3be8678215..4ee9bc2625 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Looking for test workflow entry points:                                                     +'tests/subworkflows/nf-core/bam_stats_samtools/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_bam_stats_samtools_single_end' +INFO     Running 'bam_stats_samtools' test with command:                                             +nextflow run ./tests/subworkflows/nf-core/bam_stats_samtools -entry  +test_bam_stats_samtools_single_end -c ./tests/config/nextflow.config --outdir  +/tmp/tmp2v5sjybb -work-dir /tmp/tmp8zk7tbes diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index 3454ab8a62..0d215e24f7 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + - + - - - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - + + + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +CRITICAL Could not open 'tests/config/pytest_modules.yml' file!                                      From 6b06cccfd1de05f5d96c7ec187eeca845ac2825e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Jun 2023 13:30:14 +0200 Subject: [PATCH 244/249] remove cd modules --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cc7fdefafe..fd7221f5d0 100644 --- a/README.md +++ b/README.md @@ -1185,7 +1185,7 @@ working_dir: tmp/modules fake_command: nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force --> -![`cd modules && nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force`](docs/images/nf-core-subworkflows-create.svg) +![`nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force`](docs/images/nf-core-subworkflows-create.svg) ### Create a subworkflow test config file From 602e8589bb57a9aa219807eeb97e6991221d1c30 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 30 Jun 2023 11:36:27 +0000 Subject: [PATCH 245/249] Generate new screengrabs with rich-codex --- docs/images/nf-core-modules-create-test.svg | 98 +++++++-------- .../nf-core-subworkflows-create-test.svg | 108 ++++++++-------- docs/images/nf-core-subworkflows-create.svg | 119 +++++++++++------- docs/images/nf-core-subworkflows-remove.svg | 100 +++++++-------- docs/images/nf-core-subworkflows-update.svg | 72 +++++------ 5 files changed, 255 insertions(+), 242 deletions(-) diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 40edde66fe..9af81b95f8 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,84 +19,84 @@ font-weight: 700; } - .terminal-2498731265-matrix { + .terminal-1526832300-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2498731265-title { + .terminal-1526832300-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2498731265-r1 { fill: #c5c8c6 } -.terminal-2498731265-r2 { fill: #98a84b } -.terminal-2498731265-r3 { fill: #9a9b99 } -.terminal-2498731265-r4 { fill: #608ab1 } -.terminal-2498731265-r5 { fill: #d0b344 } -.terminal-2498731265-r6 { fill: #ff2c7a } -.terminal-2498731265-r7 { fill: #98729f } + .terminal-1526832300-r1 { fill: #c5c8c6 } +.terminal-1526832300-r2 { fill: #98a84b } +.terminal-1526832300-r3 { fill: #9a9b99 } +.terminal-1526832300-r4 { fill: #608ab1 } +.terminal-1526832300-r5 { fill: #d0b344 } +.terminal-1526832300-r6 { fill: #ff2c7a } +.terminal-1526832300-r7 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108,28 +108,28 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc' -INFO     Running 'fastqc' test with command:                                                         -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  -./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmp1dv_lyip -work-dir /tmp/tmporw8oqxx + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmpe8nlzgcc -work-dir /tmp/tmputkzu3j5 diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index 4ee9bc2625..9ecec3ff4d 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Looking for test workflow entry points:                                                     -'tests/subworkflows/nf-core/bam_stats_samtools/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_bam_stats_samtools_single_end' -INFO     Running 'bam_stats_samtools' test with command:                                             -nextflow run ./tests/subworkflows/nf-core/bam_stats_samtools -entry  -test_bam_stats_samtools_single_end -c ./tests/config/nextflow.config --outdir  -/tmp/tmp2v5sjybb -work-dir /tmp/tmp8zk7tbes + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Looking for test workflow entry points:                                                     +'tests/subworkflows/nf-core/bam_stats_samtools/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_bam_stats_samtools' +INFO     Running 'bam_stats_samtools' test with command:                                             +nextflow run ./tests/subworkflows/nf-core/bam_stats_samtools -entry test_bam_stats_samtools +-c ./tests/config/nextflow.config --outdir /tmp/tmporwv9usk -work-dir /tmp/tmpgshpshl3 diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index 0d215e24f7..df2311a2fd 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -CRITICAL Could not open 'tests/config/pytest_modules.yml' file!                                      + + + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Created / edited following files:                                                           +           ./subworkflows/nf-core/bam_stats_samtools/main.nf +           ./subworkflows/nf-core/bam_stats_samtools/meta.yml +           ./tests/subworkflows/nf-core/bam_stats_samtools/main.nf +           ./tests/subworkflows/nf-core/bam_stats_samtools/test.yml +           ./tests/subworkflows/nf-core/bam_stats_samtools/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 040c8d999b..055c1c378a 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - + - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -INFO     Removed files for 'rseqc/bamstat' and it's dependencies 'rseqc/bamstat'.                    -INFO     Removed files for 'rseqc/inferexperiment' and it's dependencies 'rseqc/inferexperiment'.    -INFO     Removed files for 'rseqc/innerdistance' and it's dependencies 'rseqc/innerdistance'.        -INFO     Removed files for 'rseqc/junctionannotation' and it's dependencies                          -'rseqc/junctionannotation'.                                                                 -INFO     Removed files for 'bam_rseqc' and it's dependencies 'bam_rseqc, rseqc_bamstat,  -rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation'.                      + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +INFO     Removed files for 'rseqc/bamstat' and it's dependencies 'rseqc/bamstat'.                    +INFO     Removed files for 'rseqc/inferexperiment' and it's dependencies 'rseqc/inferexperiment'.    +INFO     Removed files for 'rseqc/innerdistance' and it's dependencies 'rseqc/innerdistance'.        +INFO     Removed files for 'bam_rseqc' and it's dependencies 'bam_rseqc, rseqc_bamstat,  +rseqc_inferexperiment, rseqc_innerdistance'.                                                diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index fd0a2cd8f3..e58178193d 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-3261404162-matrix { + .terminal-3091928051-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3261404162-title { + .terminal-3091928051-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3261404162-r1 { fill: #c5c8c6 } -.terminal-3261404162-r2 { fill: #98a84b } -.terminal-3261404162-r3 { fill: #9a9b99 } -.terminal-3261404162-r4 { fill: #608ab1 } -.terminal-3261404162-r5 { fill: #d0b344 } -.terminal-3261404162-r6 { fill: #cc555a;font-weight: bold } + .terminal-3091928051-r1 { fill: #c5c8c6 } +.terminal-3091928051-r2 { fill: #98a84b } +.terminal-3091928051-r3 { fill: #9a9b99 } +.terminal-3091928051-r4 { fill: #608ab1 } +.terminal-3091928051-r5 { fill: #d0b344 } +.terminal-3091928051-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.9 - https://nf-co.re - - - -ERROR   'rseqc/junctionsaturation' + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.9 - https://nf-co.re + + + +ERROR   'rseqc/junctionannotation' From efbe400110887c053acfcd2d023de2b7980d15a4 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 30 Jun 2023 15:36:06 +0200 Subject: [PATCH 246/249] add hint to use markdown in component channel description --- nf_core/module-template/modules/meta.yml | 4 ++-- nf_core/subworkflow-template/subworkflows/meta.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 2c8197dcba..aea3c36aa3 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -30,7 +30,7 @@ input: type: map description: | Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + e.g. `[ id:'test', single_end:false ]` {% endif %} {% if not_empty_template -%} ## TODO nf-core: Delete / customise this example input @@ -49,7 +49,7 @@ output: type: map description: | Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + e.g. `[ id:'test', single_end:false ]` {% endif %} - versions: type: file diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/subworkflows/meta.yml index 4c5b454ddf..ae1689805d 100644 --- a/nf_core/subworkflow-template/subworkflows/meta.yml +++ b/nf_core/subworkflow-template/subworkflows/meta.yml @@ -17,7 +17,7 @@ input: type: map description: | Groovy Map containing sample information - e.g. [ id:'test' ] + e.g. `[ id:'test' ]` - bam: type: file description: BAM/CRAM/SAM file @@ -28,7 +28,7 @@ output: type: map description: | Groovy Map containing sample information - e.g. [ id:'test' ] + e.g. `[ id:'test' ]` - bam: type: file description: Sorted BAM/CRAM/SAM file From b70a034105f25f32c4ba00ac8aa95e567cc77dd0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 30 Jun 2023 15:39:36 +0200 Subject: [PATCH 247/249] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77196dd428..d385fa0445 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Add functions to dynamically include pipeline tool citations in MultiQC methods description section for better reporting. ([#2326](https://github.com/nf-core/tools/pull/2326)) - Remove `--tracedir` parameter ([#2290](https://github.com/nf-core/tools/pull/2290)) - Incorrect config parameter warnings when customising pipeline template ([#2333](https://github.com/nf-core/tools/pull/2333)) +- Use markdown syntax in the description for the meta map channels ([#23585](https://github.com/nf-core/tools/pull/2358)) ### Download From 7c627003583935d90c173595ab8dd8229823deb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 30 Jun 2023 15:51:31 +0200 Subject: [PATCH 248/249] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d385fa0445..6a55989925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ - Add functions to dynamically include pipeline tool citations in MultiQC methods description section for better reporting. ([#2326](https://github.com/nf-core/tools/pull/2326)) - Remove `--tracedir` parameter ([#2290](https://github.com/nf-core/tools/pull/2290)) - Incorrect config parameter warnings when customising pipeline template ([#2333](https://github.com/nf-core/tools/pull/2333)) -- Use markdown syntax in the description for the meta map channels ([#23585](https://github.com/nf-core/tools/pull/2358)) +- Use markdown syntax in the description for the meta map channels ([#2358](https://github.com/nf-core/tools/pull/2358)) ### Download From 464f2fb63ac433d7a9c0512191edac2a5f40b024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 30 Jun 2023 14:25:45 +0000 Subject: [PATCH 249/249] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82ad6172a6..9386e27f5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - Refactored the CLI parameters related to container images. Although downloading other images than those of the Singularity/Apptainer container system is not supported for the time being, a generic name for the parameters seemed preferable. So the new parameter `--singularity-cache-index` introduced in [#2247](https://github.com/nf-core/tools/pull/2247) has been renamed to `--container-cache-index` prior to release ([#2336](https://github.com/nf-core/tools/pull/2336)). - To address issue [#2311](https://github.com/nf-core/tools/issues/2311), a new parameter `--container-library` was created allowing to specify the container library (registry) from which container images in OCI format (Docker) should be pulled ([#2336](https://github.com/nf-core/tools/pull/2336)). - Container detection in configs was improved. This allows for DSL2-like container definitions inside the container parameter value provided to process scopes [#2346](https://github.com/nf-core/tools/pull/2346). +- Add apptainer to the list of false positve container strings ([#2353](https://github.com/nf-core/tools/pull/2353)). #### Updated CLI parameters