From 98b7da08d9c49b1090719c543dcb90e25fcca108 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 28 Feb 2024 14:22:32 +0100 Subject: [PATCH 001/142] upgrade to python 3.12 --- .github/ISSUE_TEMPLATE/bug_report.yml | 4 ++-- .github/workflows/changelog.yml | 6 +++--- .github/workflows/create-lint-wf.yml | 4 ++-- .github/workflows/create-test-lint-wf-template.yml | 4 ++-- .github/workflows/create-test-wf.yml | 4 ++-- .github/workflows/deploy-pypi.yml | 4 ++-- .github/workflows/fix-linting.yml | 2 +- .github/workflows/lint-code.yml | 4 ++-- .github/workflows/pytest.yml | 6 +++--- .github/workflows/sync.yml | 4 ++-- Dockerfile | 2 +- README.md | 2 +- .../.github/workflows/download_pipeline.yml | 2 +- nf_core/pipeline-template/.github/workflows/fix-linting.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 6 +++--- 15 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8fdd2bd7e1..5043b37acc 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -37,5 +37,5 @@ body: * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ - * Version of nf-core/tools _(eg. 1.1, 1.5, 1.8.2)_ - * Python version _(eg. 3.10, 3.11)_ + * Version of nf-core/tools _(eg. 1.10, 1.12.1, 1.13)_ + * Python version _(eg. 3.11, 3.12)_ diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 6316d62f33..88d0be2eb1 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install packages run: | @@ -62,10 +62,10 @@ jobs: git diff --exit-code ${GITHUB_WORKSPACE}/CHANGELOG.md || echo "changed=YES" >> $GITHUB_ENV echo "File changed: ${{ env.changed }}" - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index e69229c051..0162dc6ec0 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -49,10 +49,10 @@ jobs: name: Check out source-code repository # Set up nf-core/tools - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" cache: pip - name: Install python dependencies diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 6f27236462..3f1bc362f3 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -61,10 +61,10 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 0166931be7..e82faae592 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -48,10 +48,10 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index e53d2f2f5a..ca698dad1f 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 95a03c70fe..3e6c4d4ef6 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -34,7 +34,7 @@ jobs: # Install and run pre-commit - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install pre-commit run: pip install pre-commit diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 8ed52a0582..53151a8989 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -20,10 +20,10 @@ jobs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5faeef49da..c749458dbe 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -38,7 +38,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.8", "3.12"] runner: ["ubuntu-latest"] include: - python-version: "3.8" @@ -173,12 +173,12 @@ jobs: cd pytest - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 env: AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/ with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install dependencies diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index d89e255bfb..14eb589360 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -56,10 +56,10 @@ jobs: path: nf-core/${{ matrix.pipeline }} fetch-depth: "0" - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/Dockerfile b/Dockerfile index 62431be140..6ca7e9a67f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim@sha256:ce81dc539f0aedc9114cae640f8352fad83d37461c24a3615b01f081d0c0583a +FROM python:3.12-slim LABEL authors="phil.ewels@scilifelab.se,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for the nfcore tools" diff --git a/README.md b/README.md index cf0b01d210..833fad626f 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ conda install nf-core Alternatively, you can create a new environment with both nf-core/tools and nextflow: ```bash -conda create --name nf-core python=3.11 nf-core nextflow +conda create --name nf-core python=3.12 nf-core nextflow conda activate nf-core ``` diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 20b811ab2b..4017fbdc2d 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 with: diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 28e6605b96..e976b9f09c 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -34,7 +34,7 @@ jobs: # Install and run pre-commit - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install pre-commit run: pip install pre-commit diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 3ec259b5ed..ea740d15a3 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -16,10 +16,10 @@ jobs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit @@ -39,7 +39,7 @@ jobs: - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - name: Install dependencies From ecc697ea8c98fddf3e0b0ab0d424916f88848a98 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 28 Feb 2024 13:27:11 +0000 Subject: [PATCH 002/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fe084d81a..7c6be5ca44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Remove obsolete editor settings in `devcontainer.json` and `gitpod.yml` ([#2795](https://github.com/nf-core/tools/pull/2795)) +- Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) ### Linting From 820510f9abd03c1d958903a0a2088fc4d38a9b51 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 18 Mar 2024 13:00:24 +0000 Subject: [PATCH 003/142] Fix the check for base dir to ensure we check if the supplied dir has a config before we start iterating up the tree --- nf_core/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 9261388a39..8319c8d31b 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1047,12 +1047,13 @@ def load_tools_config(directory: Union[str, Path] = "."): def determine_base_dir(directory="."): base_dir = start_dir = Path(directory).absolute() - while base_dir != base_dir.parent: + # Only iterate up the tree if the start dir doesn't have a config + while not get_first_available_path(base_dir, CONFIG_PATHS) and base_dir != base_dir.parent: base_dir = base_dir.parent config_fn = get_first_available_path(base_dir, CONFIG_PATHS) if config_fn: - return directory if base_dir == start_dir else base_dir - return directory + break + return directory if base_dir == start_dir else base_dir def get_first_available_path(directory, paths): From 68ab4feb415843f18209f7d02acdad8165f59492 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 19 Mar 2024 10:01:35 +0000 Subject: [PATCH 004/142] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0d37627a..a47fda7270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - Update python:3.11-slim Docker digest to a2eb07f ([#2847](https://github.com/nf-core/tools/pull/2847)) - Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 ([#2850](https://github.com/nf-core/tools/pull/2850)) +- Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From c8f3dc0b02d0b33a5c1c8fb8eed9e9034054b710 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Tue, 19 Mar 2024 10:04:10 +0000 Subject: [PATCH 005/142] Add nf_core_version to .nf-core.yml --- nf_core/pipeline-template/.nf-core.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/.nf-core.yml b/nf_core/pipeline-template/.nf-core.yml index 3805dc81c1..e8140bfb12 100644 --- a/nf_core/pipeline-template/.nf-core.yml +++ b/nf_core/pipeline-template/.nf-core.yml @@ -1 +1,2 @@ repository_type: pipeline +nf_core_version: "{{ nf_core_version }}" From c1d9dbd09f918050e14951ade2d8a7354f476512 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Tue, 19 Mar 2024 10:10:49 +0000 Subject: [PATCH 006/142] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3189edaa..d7b58a0319 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Fix topic extraction step for hashtags in toots ([#2810](https://github.com/nf-core/tools/pull/2810)) - Update modules and subworkflows in the template ([#2811](https://github.com/nf-core/tools/pull/2811)) - Unpin setup-nextflow and action-tower-launch ([#2806](https://github.com/nf-core/tools/pull/2806)) +- Add nf-core-version to `.nf-core.yml` ([#2874](https://github.com/nf-core/tools/pull/2874)) ### Download From 05083f662f7abf9c7a25e82d35a059edcbb2ec67 Mon Sep 17 00:00:00 2001 From: Weronika Sosnowska Date: Tue, 19 Mar 2024 11:41:52 +0100 Subject: [PATCH 007/142] Add type of values to the error description for schema errors --- nf_core/schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index df04dc5a1e..ceb17a2e29 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -807,8 +807,11 @@ def add_schema_found_configs(self): elif p_key in self.schema_defaults and (s_def := self.schema_defaults[p_key]) != ( p_def := self.build_schema_param(p_val).get("default") ): + p_type = self.build_schema_param(p_val).get("type") + s_type = self.schema_types[p_key] if self.no_prompts or Confirm.ask( - f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{s_def}' | config: '{p_def}'). " + f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. " + f"(schema: '{s_def}' {s_type} | config: '{p_def}' {p_type}). " "[blue]Update pipeline schema?" ): s_key_def = s_key + ("default",) From d1a384be88393c74ce6fb00d7314005745987144 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 19 Mar 2024 11:47:37 +0000 Subject: [PATCH 008/142] fix: ensure path object converted to string before stripping quotes --- nf_core/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 373f8bbaa1..ef45b201fc 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -582,7 +582,7 @@ def make_skeleton_schema(self): ) schema_template = env.get_template("nextflow_schema.json") template_vars = { - "name": self.pipeline_manifest.get("name", Path(self.schema_filename).parent).strip("'"), + "name": self.pipeline_manifest.get("name", str(Path(self.schema_filename).parent)).strip("'"), "description": self.pipeline_manifest.get("description", "").strip("'"), } self.schema = json.loads(schema_template.render(template_vars)) From e7f5a48a9204c590159a30ea428d5c8294a9b4f4 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 19 Mar 2024 11:49:27 +0000 Subject: [PATCH 009/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a47fda7270..449edff6ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ - Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 ([#2850](https://github.com/nf-core/tools/pull/2850)) - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) +- fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 7dc7bd7abc2b246f37f5e051d2ced2763d2ac312 Mon Sep 17 00:00:00 2001 From: Weronika Sosnowska Date: Tue, 19 Mar 2024 12:53:14 +0100 Subject: [PATCH 010/142] Detect type from actual value --- nf_core/schema.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 5131650b0e..99a72d7bb5 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -799,11 +799,8 @@ def add_schema_found_configs(self): elif p_key in self.schema_defaults and (s_def := self.schema_defaults[p_key]) != ( p_def := self.build_schema_param(p_val).get("default") ): - p_type = self.build_schema_param(p_val).get("type") - s_type = self.schema_types[p_key] if self.no_prompts or Confirm.ask( - f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. " - f"(schema: '{s_def}' {s_type} | config: '{p_def}' {p_type}). " + f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{s_def}' {type(s_def)} | config: '{p_def}' {type(p_def)}). " "[blue]Update pipeline schema?" ): s_key_def = s_key + ("default",) From 64a164a979898bf98691bab99539bf9fdf03b07c Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:01:46 +0000 Subject: [PATCH 011/142] fix: add word_wrap to Syntax object for console printing to ensure schema docs output is not truncated --- nf_core/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 373f8bbaa1..a1b595d5b0 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -494,7 +494,7 @@ def print_documentation( if not output_fn: console = rich.console.Console() - console.print("\n", Syntax(prettified_docs, format), "\n") + console.print("\n", Syntax(prettified_docs, format, word_wrap=True), "\n") else: if Path(output_fn).exists() and not force: log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") From fa6504b5532320b4acf2006f9aff4c5ade09b8ba Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 19 Mar 2024 12:03:42 +0000 Subject: [PATCH 012/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a47fda7270..a1e72ababd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ - Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 ([#2850](https://github.com/nf-core/tools/pull/2850)) - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) +- Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From b425c96f32de017a65cfda61b8a9e8ceb9706d6e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Mar 2024 14:27:53 +0100 Subject: [PATCH 013/142] Remove pyproject.toml --- nf_core/pipeline-template/pyproject.toml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 nf_core/pipeline-template/pyproject.toml diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml deleted file mode 100644 index 56110621e7..0000000000 --- a/nf_core/pipeline-template/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff. -# Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. -[tool.ruff] -line-length = 120 -target-version = "py38" -cache-dir = "~/.cache/ruff" - -[tool.ruff.lint] -select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] - -[tool.ruff.lint.isort] -known-first-party = ["nf_core"] - -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["E402", "F401"] From c083f641e3c6fc594afd63ed6ba676ca57f0729b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Mar 2024 15:51:35 +0100 Subject: [PATCH 014/142] Remove pyproject.toml from linting --- nf_core/lint/files_exist.py | 44 ++++++++++++++++----------------- nf_core/lint/files_unchanged.py | 3 +-- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 5d62a23bf8..ae105d8d2c 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -66,31 +66,31 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: conf/igenomes.config .github/workflows/awstest.yml .github/workflows/awsfulltest.yml - pyproject.toml Files that *must not* be present, due to being renamed or removed in the template: .. code-block:: bash - Singularity - parameters.settings.json - pipeline_template.yml # saving information in .nf-core.yml - .nf-core.yaml # NB: Should be yml, not yaml - bin/markdown_to_html.r - conf/aws.config - .github/workflows/push_dockerhub.yml .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/feature_request.md - docs/images/nf-core-PIPELINE_logo.png + .github/workflows/push_dockerhub.yml .markdownlint.yml + .nf-core.yaml # NB: Should be yml, not yaml .yamllint.yml + bin/markdown_to_html.r + conf/aws.config + docs/images/nf-core-PIPELINE_logo.png lib/Checks.groovy lib/Completion.groovy - lib/Workflow.groovy - lib/WorkflowPIPELINE.groovy lib/NfcoreTemplate.groovy lib/Utils.groovy + lib/Workflow.groovy lib/WorkflowMain.groovy + lib/WorkflowPIPELINE.groovy + parameters.settings.json + pipeline_template.yml # saving information in .nf-core.yml + pyproject.toml + Singularity Files that *should not* be present: @@ -176,30 +176,30 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: [Path(".github", "workflows", "awstest.yml")], [Path(".github", "workflows", "awsfulltest.yml")], [Path("modules.json")], - [Path("pyproject.toml")], ] # List of strings. Fails / warns if any of the strings exist. files_fail_ifexists = [ - Path("Singularity"), - Path("parameters.settings.json"), - Path("pipeline_template.yml"), # saving information in .nf-core.yml - Path(".nf-core.yaml"), # yml not yaml - Path("bin", "markdown_to_html.r"), - Path("conf", "aws.config"), - Path(".github", "workflows", "push_dockerhub.yml"), Path(".github", "ISSUE_TEMPLATE", "bug_report.md"), Path(".github", "ISSUE_TEMPLATE", "feature_request.md"), - Path("docs", "images", f"nf-core-{short_name}_logo.png"), + Path(".github", "workflows", "push_dockerhub.yml"), Path(".markdownlint.yml"), + Path(".nf-core.yaml"), # yml not yaml Path(".yamllint.yml"), + Path("bin", "markdown_to_html.r"), + Path("conf", "aws.config"), + Path("docs", "images", f"nf-core-{short_name}_logo.png"), Path("lib", "Checks.groovy"), Path("lib", "Completion.groovy"), - Path("lib", "Workflow.groovy"), + Path("lib", "NfcoreTemplate.groovy"), Path("lib", "Utils.groovy"), + Path("lib", "Workflow.groovy"), Path("lib", "WorkflowMain.groovy"), - Path("lib", "NfcoreTemplate.groovy"), Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy"), + Path("parameters.settings.json"), + Path("pipeline_template.yml"), # saving information in .nf-core.yml + Path("pyproject.toml"), + Path("Singularity"), ] files_warn_ifexists = [Path(".travis.yml")] files_fail_ifinconfig: List[Tuple[Path, Dict[str, str]]] = [ diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 3a3a0cb74a..1cd1f7fdb1 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -47,7 +47,6 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: .gitignore .prettierignore - pyproject.toml .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting @@ -106,7 +105,7 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: [Path("docs", "README.md")], ] files_partial = [ - [Path(".gitignore"), Path(".prettierignore"), Path("pyproject.toml")], + [Path(".gitignore"), Path(".prettierignore")], ] # Only show error messages from pipeline creation From 1dbc1302916cd2b23b7bd79e3601a21bf7f6feb2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Mar 2024 15:52:17 +0100 Subject: [PATCH 015/142] Add to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b31cc30ec..e6ab87636d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Set topic variable correctly in the mastodon announcement ([#2848](https://github.com/nf-core/tools/pull/2848)) - Add a cleanup action to `download_pipeline.yml` to fix failures caused by inadequate storage space on the runner ([#2849](https://github.com/nf-core/tools/pull/2849)) - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) +- Remove `pyproject.toml` from template root ### Linting From d82d28ec457aa21edaf9abe6b1293cf6e56eb8bd Mon Sep 17 00:00:00 2001 From: leomrtns Date: Tue, 19 Mar 2024 15:30:47 +0000 Subject: [PATCH 016/142] error message if wrong GITHUB_TOKEN leads to 401 --- nf_core/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/utils.py b/nf_core/utils.py index e1778b55b3..3feeae7824 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -453,6 +453,7 @@ def __init__(self): # pylint: disable=super-init-not-called self.auth_mode = None self.return_ok = [200, 201] self.return_retry = [403] + self.return_unauthorised = [401] self.has_init = False def lazy_init(self): @@ -545,6 +546,8 @@ def safe_get(self, url): raise e else: return r + elif request.status_code in self.return_unauthorised: + raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN") return request From 738323b86969e24602538c894fa02fc3aeb0adae Mon Sep 17 00:00:00 2001 From: leomrtns Date: Tue, 19 Mar 2024 16:05:07 +0000 Subject: [PATCH 017/142] merged with dev instead of master; ruff formatted; changelog updated --- CHANGELOG.md | 1 + nf_core/utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b31cc30ec..94cc5cb407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ ### Download - Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. +- Better error message when GITHUB_TOKEN exists but is wrong/outdated ### Components diff --git a/nf_core/utils.py b/nf_core/utils.py index 5307363ce3..2c3f954f99 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -548,7 +548,7 @@ def safe_get(self, url): else: return r elif request.status_code in self.return_unauthorised: - raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN") + raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN") return request From 81d55097df2a38eb19abd83de33ab378f398f5d9 Mon Sep 17 00:00:00 2001 From: leomrtns Date: Tue, 19 Mar 2024 16:21:23 +0000 Subject: [PATCH 018/142] suggested change by Matthias --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 2c3f954f99..461f5a9bad 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -548,7 +548,7 @@ def safe_get(self, url): else: return r elif request.status_code in self.return_unauthorised: - raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN") + raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") return request From 63d5de8fe69b2ede6eb63a5693d65747c47a37a4 Mon Sep 17 00:00:00 2001 From: Emilio Palumbo Date: Tue, 19 Mar 2024 18:07:30 +0100 Subject: [PATCH 019/142] Rename main.nf.test template for modules and subworkflows --- nf_core/components/create.py | 2 +- nf_core/module-template/tests/{main.nf.test => main.nf.test.j2} | 0 .../tests/{main.nf.test => main.nf.test.j2} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename nf_core/module-template/tests/{main.nf.test => main.nf.test.j2} (100%) rename nf_core/subworkflow-template/tests/{main.nf.test => main.nf.test.j2} (100%) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index d2169e3a72..6c9c01b496 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -394,7 +394,7 @@ def _get_component_dirs(self): if self.component_type == "modules": file_paths["environment.yml"] = component_dir / "environment.yml" file_paths["tests/tags.yml"] = component_dir / "tests" / "tags.yml" - file_paths["tests/main.nf.test"] = component_dir / "tests" / "main.nf.test" + file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" return file_paths diff --git a/nf_core/module-template/tests/main.nf.test b/nf_core/module-template/tests/main.nf.test.j2 similarity index 100% rename from nf_core/module-template/tests/main.nf.test rename to nf_core/module-template/tests/main.nf.test.j2 diff --git a/nf_core/subworkflow-template/tests/main.nf.test b/nf_core/subworkflow-template/tests/main.nf.test.j2 similarity index 100% rename from nf_core/subworkflow-template/tests/main.nf.test rename to nf_core/subworkflow-template/tests/main.nf.test.j2 From 7364a536661b1f3d2053fb42c6df0755e2124fb0 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 08:48:57 +0000 Subject: [PATCH 020/142] Add version check in snapshot file --- nf_core/modules/lint/module_tests.py | 16 ++++++++++++++++ nf_core/subworkflows/lint/subworkflow_tests.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index b1a611d70a..b227f684bd 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -107,6 +107,22 @@ def module_tests(_, module: NFCoreComponent): snap_file, ) ) + if "versions" in str(snap_content[test_name]): + module.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, + ) + ) + else: + module.failed.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, + ) + ) except json.decoder.JSONDecodeError as e: module.failed.append( ( diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 796a56d018..0c0589a1e2 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -114,6 +114,22 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): snap_file, ) ) + if "versions" in str(snap_content[test_name]): + subworkflow.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, + ) + ) + else: + subworkflow.warned.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, + ) + ) except json.decoder.JSONDecodeError as e: subworkflow.failed.append( ( From 8a5c964f2b9e9b407799bbe7981e7771b5d25e5f Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 20 Mar 2024 09:00:15 +0000 Subject: [PATCH 021/142] [automated] Fix code linting --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 461f5a9bad..4271e971a1 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -548,7 +548,7 @@ def safe_get(self, url): else: return r elif request.status_code in self.return_unauthorised: - raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") + raise RuntimeError("GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") return request From eccf0db977aba848d4b4e77d8075fe0cee2287cd Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 09:29:51 +0000 Subject: [PATCH 022/142] update changelog.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94cc5cb407..9177060b09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Linting - Only match assignments of params in `main.nf` and not references like `params.aligner == ` ([#2833](https://github.com/nf-core/tools/pull/2833)) +- Include test for presence of versions in snapshot ([#2888](https://github.com/nf-core/tools/pull/2888)) ### Download From 44bf8005cba0c76ded34062a698ae7a55207aee7 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 10:27:32 +0000 Subject: [PATCH 023/142] adding pytests --- tests/modules/lint.py | 21 +++++++++++++++++++++ tests/subworkflows/lint.py | 19 +++++++++++++++++++ tests/test_modules.py | 1 + tests/test_subworkflows.py | 1 + 4 files changed, 42 insertions(+) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 9bd280ddd8..5e3bcfbd30 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -642,3 +642,24 @@ def test_nftest_failing_linting(self): assert module_lint.failed[2].lint_test == "test_main_tags" assert "kallisto/index" in module_lint.failed[2].message assert module_lint.failed[3].lint_test == "test_tags_yml" + + +def test_modules_absent_version(self): + """Test linting a nf-test module if the versions is absent in the snapshot file `""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap")) as fh: + content = fh.read() + new_content = content.replace("versions", "foo") + with open( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap"), "w" + ) as fh: + fh.write(new_content) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + with open( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap"), "w" + ) as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_snap_versions" diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index b89b7b78ce..7da5f1640f 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -181,3 +181,22 @@ def test_subworkflows_lint_capitalization_fail(self): # cleanup self.subworkflow_remove.remove("bam_stats_samtools", force=True) + + +def test_subworkflows_absent_version(self): + """Test linting a nf-test module if the versions is absent in the snapshot file `""" + self.subworkflow_install.install("fastq_align_dna") + with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_dna", "test", "main.nf.test.snap")) as fh: + content = fh.read() + new_content = content.replace("versions", "foo") + with open( + Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_dna", "test", "main.nf.test.snap"), "w" + ) as fh: + fh.write(new_content) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, subworkflow="fastq_align_dna") + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert any([x.lint_test == "test_snap_versions" for x in subworkflow_lint.warned]) diff --git a/tests/test_modules.py b/tests/test_modules.py index 944a09f670..da336aa67a 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -179,6 +179,7 @@ def test_modulesrepo_class(self): test_modules_install_trimgalore_twice, ) from .modules.lint import ( # type: ignore[misc] + test_modules_absent_version, test_modules_environment_yml_file_doesnt_exists, test_modules_environment_yml_file_name_mismatch, test_modules_environment_yml_file_not_array, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 6163faa7a9..775a08547f 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -119,6 +119,7 @@ def tearDown(self): test_subworkflows_install_tracking_added_super_subworkflow, ) from .subworkflows.lint import ( # type: ignore[misc] + test_subworkflows_absent_version, test_subworkflows_lint, test_subworkflows_lint_capitalization_fail, test_subworkflows_lint_empty, From c77cfc87613f197d14048ee964b0273401f47bbc Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 11:04:09 +0000 Subject: [PATCH 024/142] adding additional logic for split up snapshots --- nf_core/modules/lint/module_tests.py | 2 +- nf_core/subworkflows/lint/subworkflow_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index b227f684bd..52cbaf04e5 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -107,7 +107,7 @@ def module_tests(_, module: NFCoreComponent): snap_file, ) ) - if "versions" in str(snap_content[test_name]): + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): module.passed.append( ( "test_snap_versions", diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 0c0589a1e2..30d4296aac 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -114,7 +114,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): snap_file, ) ) - if "versions" in str(snap_content[test_name]): + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): subworkflow.passed.append( ( "test_snap_versions", From 099c01dbfd4ff86c1a1c8235067e07d5ca0292d9 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 11:41:57 +0000 Subject: [PATCH 025/142] setup .nf-core.yml lint --- nf_core/lint/nfcore_yml.py | 61 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 nf_core/lint/nfcore_yml.py diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py new file mode 100644 index 0000000000..4b165c78e4 --- /dev/null +++ b/nf_core/lint/nfcore_yml.py @@ -0,0 +1,61 @@ +def nfcore_yml(self): + """Repository ``.nf-core.yml`` tests + + The ``.nf-core.yml`` contains metadata for nf-core tools to correctly apply its features. + + * repository type: + + * Check that the repository type is set. + + * nf core version: + + * Check if the nf-core version is set to the latest version. + + """ + passed = [] + warned = [] + failed = [] + + # Remove field that should be ignored according to the linting config + # ignore_configs = self.lint_config.get(".nf-core", []) + + # with open(os.path.join(self.wf_path, ".nf-core.yml")) as fh: + # content = fh.read() + + # if "nextflow_badge" not in ignore_configs: + # # Check that there is a readme badge showing the minimum required version of Nextflow + # # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) + # # and that it has the correct version + # nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" + # match = re.search(nf_badge_re, content) + # if match: + # nf_badge_version = match.group(1).strip("'\"") + # try: + # if nf_badge_version != self.minNextflowVersion: + # raise AssertionError() + # except (AssertionError, KeyError): + # failed.append( + # f"README Nextflow minimum version badge does not match config. Badge: `{nf_badge_version}`, " + # f"Config: `{self.minNextflowVersion}`" + # ) + # else: + # passed.append( + # f"README Nextflow minimum version badge matched config. Badge: `{nf_badge_version}`, " + # f"Config: `{self.minNextflowVersion}`" + # ) + # else: + # warned.append("README did not have a Nextflow minimum version badge.") + + # if "zenodo_doi" not in ignore_configs: + # # Check that zenodo.XXXXXXX has been replaced with the zendo.DOI + # zenodo_re = r"/zenodo\.X+" + # match = re.search(zenodo_re, content) + # if match: + # warned.append( + # "README contains the placeholder `zenodo.XXXXXXX`. " + # "This should be replaced with the zenodo doi (after the first release)." + # ) + # else: + # passed.append("README Zenodo placeholder was replaced with DOI.") + + return {"passed": passed, "warned": warned, "failed": failed} From 2ee4a204f0bcb4ee12aaef73385c7a075edf7410 Mon Sep 17 00:00:00 2001 From: Weronika Sosnowska Date: Wed, 20 Mar 2024 12:49:44 +0100 Subject: [PATCH 026/142] Apply suggestion from review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 8aa863a4ee..4096c80f93 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -800,7 +800,7 @@ def add_schema_found_configs(self): p_def := self.build_schema_param(p_val).get("default") ): if self.no_prompts or Confirm.ask( - f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{s_def}' {type(s_def)} | config: '{p_def}' {type(p_def)}). " + f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{type(s_def)}: {s_def}' | config: '{type(p_def)}: {p_def}'). " "[blue]Update pipeline schema?" ): s_key_def = s_key + ("default",) From 07bb48295e9967da5ef59ac7f81906e1bde16bbb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 03:07:27 +0000 Subject: [PATCH 027/142] Update dawidd6/action-download-artifact digest to 09f2f74 --- nf_core/pipeline-template/.github/workflows/linting_comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 67ef7b534d..ea408fd6f8 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@a430ac5786b39ad5869da25a98130624d2ce340c # v3 + uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3 with: workflow: linting.yml workflow_conclusion: completed From 353fc04e90591bf79a22b07f12716c4c974e8443 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 13:26:43 +0000 Subject: [PATCH 028/142] Add nfcore_yml linting module*** --- nf_core/lint/__init__.py | 2 + nf_core/lint/nfcore_yml.py | 93 ++++++++++++++++++++++---------------- tests/lint/nfcore_yml.py | 53 ++++++++++++++++++++++ tests/test_lint.py | 5 ++ 4 files changed, 113 insertions(+), 40 deletions(-) create mode 100644 tests/lint/nfcore_yml.py diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index be9ac183a6..a14ac15691 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -206,6 +206,7 @@ class PipelineLint(nf_core.utils.Pipeline): from .modules_structure import modules_structure # type: ignore[misc] from .multiqc_config import multiqc_config # type: ignore[misc] from .nextflow_config import nextflow_config # type: ignore[misc] + from .nfcore_yml import nfcore_yml # type: ignore[misc] from .pipeline_name_conventions import ( # type: ignore[misc] pipeline_name_conventions, ) @@ -264,6 +265,7 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "nfcore_yml", ] + (["version_consistency"] if release_mode else []) def _load(self): diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index 4b165c78e4..6d9f664871 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -1,3 +1,11 @@ +import os +import re + +from nf_core import __version__ + +REPOSITORY_TYPES = ["pipeline", "modules"] + + def nfcore_yml(self): """Repository ``.nf-core.yml`` tests @@ -15,47 +23,52 @@ def nfcore_yml(self): passed = [] warned = [] failed = [] + ignored = [] # Remove field that should be ignored according to the linting config - # ignore_configs = self.lint_config.get(".nf-core", []) - - # with open(os.path.join(self.wf_path, ".nf-core.yml")) as fh: - # content = fh.read() - - # if "nextflow_badge" not in ignore_configs: - # # Check that there is a readme badge showing the minimum required version of Nextflow - # # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) - # # and that it has the correct version - # nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" - # match = re.search(nf_badge_re, content) - # if match: - # nf_badge_version = match.group(1).strip("'\"") - # try: - # if nf_badge_version != self.minNextflowVersion: - # raise AssertionError() - # except (AssertionError, KeyError): - # failed.append( - # f"README Nextflow minimum version badge does not match config. Badge: `{nf_badge_version}`, " - # f"Config: `{self.minNextflowVersion}`" - # ) - # else: - # passed.append( - # f"README Nextflow minimum version badge matched config. Badge: `{nf_badge_version}`, " - # f"Config: `{self.minNextflowVersion}`" - # ) - # else: - # warned.append("README did not have a Nextflow minimum version badge.") - - # if "zenodo_doi" not in ignore_configs: - # # Check that zenodo.XXXXXXX has been replaced with the zendo.DOI - # zenodo_re = r"/zenodo\.X+" - # match = re.search(zenodo_re, content) - # if match: - # warned.append( - # "README contains the placeholder `zenodo.XXXXXXX`. " - # "This should be replaced with the zenodo doi (after the first release)." - # ) - # else: - # passed.append("README Zenodo placeholder was replaced with DOI.") + ignore_configs = self.lint_config.get(".nf-core", []) + + try: + with open(os.path.join(self.wf_path, ".nf-core.yml")) as fh: + content = fh.read() + except FileNotFoundError: + with open(os.path.join(self.wf_path, ".nf-core.yaml")) as fh: + content = fh.read() + + if "repository_type" not in ignore_configs: + # Check that the repository type is set in the .nf-core.yml + repo_type_re = r"repository_type: (.+)" + match = re.search(repo_type_re, content) + if match: + repo_type = match.group(1) + if repo_type not in REPOSITORY_TYPES: + failed.append( + f"Repository type in .nf-core.yml is not valid. " + f"Should be one of {', '.join(REPOSITORY_TYPES)} but was {repo_type}" + ) + else: + passed.append(f"Repository type in .nf-core.yml is valid: {repo_type}") + else: + warned.append("Repository type not set in .nf-core.yml") + else: + ignored.append(".nf-core.yml variable ignored 'repository_type'") + + if "nf_core_version" not in ignore_configs: + # Check that the nf-core version is set in the .nf-core.yml + nf_core_version_re = r"nf_core_version: (.+)" + match = re.search(nf_core_version_re, content) + if match: + nf_core_version = match.group(1) + if nf_core_version != __version__ and "dev" not in __version__: + warned.append( + f"nf-core version in .nf-core.yml is not set to the latest version. " + f"Should be {__version__} but was {nf_core_version}" + ) + else: + passed.append(f"nf-core version in .nf-core.yml is set to the latest version: {nf_core_version}") + else: + warned.append("nf-core version not set in .nf-core.yml") + else: + ignored.append(".nf-core.yml variable ignored 'nf_core_version'") return {"passed": passed, "warned": warned, "failed": failed} diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py new file mode 100644 index 0000000000..ea5fa8dee5 --- /dev/null +++ b/tests/lint/nfcore_yml.py @@ -0,0 +1,53 @@ +import re +from pathlib import Path + +import nf_core.create +import nf_core.lint + + +def test_nfcore_yml_pass(self): + """Lint test: nfcore_yml - PASS""" + self.lint_obj._load() + results = self.lint_obj.nfcore_yml() + + assert "Repository type in .nf-core.yml is valid" in str(results["passed"]) + assert "nf-core version in .nf-core.yml is set to the latest version" in str(results["passed"]) + assert len(results.get("warned", [])) == 0 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 + + +def test_nfcore_yml_fail_repo_type(self): + """Lint test: nfcore_yml - FAIL - repository type not set""" + new_pipeline = self._make_pipeline_copy() + nf_core_yml = Path(new_pipeline) / ".nf-core.yml" + with open(nf_core_yml) as fh: + content = fh.read() + new_content = content.replace("repository_type: pipeline", "repository_type: foo") + with open(nf_core_yml, "w") as fh: + fh.write(new_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + results = lint_obj.nfcore_yml() + assert "Repository type in .nf-core.yml is not valid." in str(results["failed"]) + assert len(results.get("warned", [])) == 0 + assert len(results.get("passed", [])) >= 0 + assert len(results.get("ignored", [])) == 0 + + +def test_nfcore_yml_fail_nfcore_version(self): + """Lint test: nfcore_yml - FAIL - nf-core version not set""" + new_pipeline = self._make_pipeline_copy() + nf_core_yml = Path(new_pipeline) / ".nf-core.yml" + with open(nf_core_yml) as fh: + content = fh.read() + new_content = (re.sub(r"nf-core_version: [\s]*", "nf-core_version: foo", content),) + with open(nf_core_yml, "w") as fh: + fh.write(new_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + results = lint_obj.nfcore_yml() + assert "nf-core version in .nf-core.yml is not set to the latest version." in str(results["warned"]) + assert len(results.get("failed", [])) == 0 + assert len(results.get("passed", [])) >= 0 + assert len(results.get("ignored", [])) == 0 diff --git a/tests/test_lint.py b/tests/test_lint.py index d10cef37e4..0d767dc1db 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -233,6 +233,11 @@ def test_sphinx_md_files(self): test_nextflow_config_example_pass, test_nextflow_config_missing_test_profile_failed, ) + from .lint.nfcore_yml import ( # type: ignore[misc] + test_nfcore_yml_fail_nfcore_version, + test_nfcore_yml_fail_repo_type, + test_nfcore_yml_pass, + ) from .lint.template_strings import ( # type: ignore[misc] test_template_strings, test_template_strings_ignore_file, From 907df3bd8d95affb5fc01d90b31c7f45dd1b8582 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 13:47:36 +0000 Subject: [PATCH 029/142] Update nf-core_version in .nf-core.yml file --- tests/lint/nfcore_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py index ea5fa8dee5..4c36d27c25 100644 --- a/tests/lint/nfcore_yml.py +++ b/tests/lint/nfcore_yml.py @@ -41,7 +41,7 @@ def test_nfcore_yml_fail_nfcore_version(self): nf_core_yml = Path(new_pipeline) / ".nf-core.yml" with open(nf_core_yml) as fh: content = fh.read() - new_content = (re.sub(r"nf-core_version: [\s]*", "nf-core_version: foo", content),) + new_content = re.sub(r"nf-core_version: [\s]*", "nf-core_version: foo", content) with open(nf_core_yml, "w") as fh: fh.write(new_content) lint_obj = nf_core.lint.PipelineLint(new_pipeline) From f6561ee78086de631f80fe9a240169fa8cee53eb Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 14:00:16 +0000 Subject: [PATCH 030/142] Fix linting test for subworkflows --- tests/subworkflows/lint.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 7da5f1640f..84ed6d4234 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -185,18 +185,20 @@ def test_subworkflows_lint_capitalization_fail(self): def test_subworkflows_absent_version(self): """Test linting a nf-test module if the versions is absent in the snapshot file `""" - self.subworkflow_install.install("fastq_align_dna") - with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_dna", "test", "main.nf.test.snap")) as fh: + self.subworkflow_install.install("bam_sort_stats_samtools") + with open( + Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_sort_stats_samtools", "test", "main.nf.test.snap") + ) as fh: content = fh.read() new_content = content.replace("versions", "foo") with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_dna", "test", "main.nf.test.snap"), "w" + Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_sort_stats_samtools", "test", "main.nf.test.snap"), "w" ) as fh: fh.write(new_content) subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, subworkflow="fastq_align_dna") + subworkflow_lint.lint(print_results=False, subworkflow="bam_sort_stats_samtools") assert len(subworkflow_lint.failed) == 0 assert len(subworkflow_lint.passed) > 0 - assert len(subworkflow_lint.warned) >= 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" assert any([x.lint_test == "test_snap_versions" for x in subworkflow_lint.warned]) From 3b2aaa9d6c6fe0464175cc52b0df3eb2094cd8ae Mon Sep 17 00:00:00 2001 From: Pieter Moris <13552343+pmoris@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:06:16 +0100 Subject: [PATCH 031/142] Make cli-provided module/subworkflow names case insensitive Uses a click callback function to normalize the name of a module/subworkflow that is provided by the user on the CLI, so that names written in uppercase still resolve to the expected lowercase name. In case no argument is provided, the callback function simply returns None, and thus the current behaviour of a user being asked to select a component via an interactive prompt is maintained. The .casefold() method is used in favour of .lower() for slightly more robust case normalization. See: https://stackoverflow.com/questions/45745661/lower-vs-casefold-in-string-matching-and-converting-to-lowercase The callback is applied to all occurrences of "tool/module/subworkflow" arguments, except those in the the "create" functions, because these already contain an internal check that prompts to user to adhere to the lowercase naming convention. Updates to CHANGELOG.md. Co-authored-by: Adam Talbot Co-authored-by: Phil Ewels Co-authored-by: Arthur Gymer <24782660+awgymer@users.noreply.github.com> --- CHANGELOG.md | 1 + nf_core/__main__.py | 36 ++++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94cc5cb407..d8e740d2af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) +- Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 20cba43e4e..dc17c6e6c1 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -103,6 +103,14 @@ def selective_traceback_hook(exctype, value, traceback): sys.excepthook = selective_traceback_hook +# Define callback function to normalize the case of click arguments, +# which is used to make the module/subworkflow names, provided by the +# user on the cli, case insensitive. +def normalize_case(ctx, param, component_name): + if component_name is not None: + return component_name.casefold() + + def run_nf_core(): # print nf-core header if environment variable is not set if os.environ.get("_NF_CORE_COMPLETE") is None: @@ -786,7 +794,7 @@ def modules_list_local(ctx, keywords, json, dir): # pylint: disable=redefined-b # nf-core modules install @modules.command("install") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -838,7 +846,7 @@ def modules_install(ctx, tool, dir, prompt, force, sha): # nf-core modules update @modules.command("update") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -930,7 +938,7 @@ def modules_update( # nf-core modules patch @modules.command() @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -967,7 +975,7 @@ def patch(ctx, tool, dir, remove): # nf-core modules remove @modules.command("remove") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1121,7 +1129,7 @@ def create_module( # nf-core modules test @modules.command("test") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1180,7 +1188,7 @@ def test_module(ctx, tool, dir, no_prompts, update, once, profile): # nf-core modules lint @modules.command("lint") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1267,7 +1275,7 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, # nf-core modules info @modules.command("info") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1306,7 +1314,7 @@ def modules_info(ctx, tool, dir): # nf-core modules bump-versions @modules.command() @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1392,7 +1400,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): # nf-core subworkflows test @subworkflows.command("test") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1519,7 +1527,7 @@ def subworkflows_list_local(ctx, keywords, json, dir): # pylint: disable=redefi # nf-core subworkflows lint @subworkflows.command("lint") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1600,7 +1608,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo # nf-core subworkflows info @subworkflows.command("info") @click.pass_context -@click.argument("tool", type=str, required=False, metavar="subworkflow name") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1639,7 +1647,7 @@ def subworkflows_info(ctx, tool, dir): # nf-core subworkflows install @subworkflows.command("install") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1697,7 +1705,7 @@ def subworkflows_install(ctx, subworkflow, dir, prompt, force, sha): # nf-core subworkflows remove @subworkflows.command("remove") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1727,7 +1735,7 @@ def subworkflows_remove(ctx, dir, subworkflow): # nf-core subworkflows update @subworkflows.command("update") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", From 2cdb907cc1e89909b2f7b8d0a74998b0bac735a6 Mon Sep 17 00:00:00 2001 From: Pieter Moris <13552343+pmoris@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:21:58 +0100 Subject: [PATCH 032/142] Rename subworkflow argument in subworkflows info command The subworkflow info command referred to the subworkflow name as "tool" in its list of arguments. It has been renamed to "subworkflow" to align with the other subworkflow commands. --- nf_core/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index dc17c6e6c1..807bc776bb 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1608,7 +1608,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo # nf-core subworkflows info @subworkflows.command("info") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1616,7 +1616,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def subworkflows_info(ctx, tool, dir): +def subworkflows_info(ctx, subworkflow, dir): """ Show developer usage information about a given subworkflow. @@ -1633,7 +1633,7 @@ def subworkflows_info(ctx, tool, dir): try: subworkflow_info = SubworkflowInfo( dir, - tool, + subworkflow, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], From 73f507049cd26aa7ec3ed768abc3eddc26a41f48 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 14:30:06 +0000 Subject: [PATCH 033/142] this feels like cheating --- tests/test_modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_modules.py b/tests/test_modules.py index da336aa67a..5f64a233f2 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -56,6 +56,7 @@ def create_modules_repo_dummy(tmp_dir): test_snap_path.touch() with open(test_snap_path, "w") as fh: fh.write('{\n "my test": {}\n}') + fh.write('{\n "versions": {}\n}') # remove "TODO" statements from main.nf main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") From 0caee7148734c4aa64b361a4ea3d6bfa9b324355 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 14:41:00 +0000 Subject: [PATCH 034/142] Add nfcore_yml.md for pipeline lint tests --- docs/api/_src/pipeline_lint_tests/nfcore_yml.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/api/_src/pipeline_lint_tests/nfcore_yml.md diff --git a/docs/api/_src/pipeline_lint_tests/nfcore_yml.md b/docs/api/_src/pipeline_lint_tests/nfcore_yml.md new file mode 100644 index 0000000000..f7e797a29c --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/nfcore_yml.md @@ -0,0 +1,5 @@ +# nfcore_yml + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.nfcore_yml +``` From ebabece0487aea0d38f9f7ac26d7e404161db943 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 15:38:37 +0000 Subject: [PATCH 035/142] Add logging module and fix nf_core_version regex --- nf_core/lint/nfcore_yml.py | 8 ++++---- tests/lint/nfcore_yml.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index 6d9f664871..64fefa50b8 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -44,7 +44,7 @@ def nfcore_yml(self): if repo_type not in REPOSITORY_TYPES: failed.append( f"Repository type in .nf-core.yml is not valid. " - f"Should be one of {', '.join(REPOSITORY_TYPES)} but was {repo_type}" + f"Should be one of [{', '.join(REPOSITORY_TYPES)}] but was {repo_type}" ) else: passed.append(f"Repository type in .nf-core.yml is valid: {repo_type}") @@ -58,8 +58,8 @@ def nfcore_yml(self): nf_core_version_re = r"nf_core_version: (.+)" match = re.search(nf_core_version_re, content) if match: - nf_core_version = match.group(1) - if nf_core_version != __version__ and "dev" not in __version__: + nf_core_version = match.group(1).strip('"') + if nf_core_version != __version__ and "dev" not in nf_core_version: warned.append( f"nf-core version in .nf-core.yml is not set to the latest version. " f"Should be {__version__} but was {nf_core_version}" @@ -71,4 +71,4 @@ def nfcore_yml(self): else: ignored.append(".nf-core.yml variable ignored 'nf_core_version'") - return {"passed": passed, "warned": warned, "failed": failed} + return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py index 4c36d27c25..74d42fe229 100644 --- a/tests/lint/nfcore_yml.py +++ b/tests/lint/nfcore_yml.py @@ -41,7 +41,7 @@ def test_nfcore_yml_fail_nfcore_version(self): nf_core_yml = Path(new_pipeline) / ".nf-core.yml" with open(nf_core_yml) as fh: content = fh.read() - new_content = re.sub(r"nf-core_version: [\s]*", "nf-core_version: foo", content) + new_content = re.sub(r"nf_core_version:.+", "nf_core_version: foo", content) with open(nf_core_yml, "w") as fh: fh.write(new_content) lint_obj = nf_core.lint.PipelineLint(new_pipeline) From faf3a8750277c7382e52e31dcdfdc8f16bf5f698 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:48:51 +0000 Subject: [PATCH 036/142] fix: assertions for called_with/called_once_with were silently returning mocks before 3.12. Broken in 3.12. --- tests/test_cli.py | 4 ++-- tests/test_sync.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 54e420f5e4..618ef0d290 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -364,7 +364,7 @@ def test_schema_lint(self, mock_get_schema_path): """Test nf-core schema lint defaults to nextflow_schema.json""" cmd = ["schema", "lint"] result = self.invoke_cli(cmd) - assert mock_get_schema_path.called_with("nextflow_schema.json") + mock_get_schema_path.assert_called_with("nextflow_schema.json") assert "nextflow_schema.json" in result.output @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") @@ -372,7 +372,7 @@ def test_schema_lint_filename(self, mock_get_schema_path): """Test nf-core schema lint accepts a filename""" cmd = ["schema", "lint", "some_other_filename"] result = self.invoke_cli(cmd) - assert mock_get_schema_path.called_with("some_other_filename") + mock_get_schema_path.assert_called_with("some_other_filename") assert "some_other_filename" in result.output assert "nextflow_schema.json" not in result.output diff --git a/tests/test_sync.py b/tests/test_sync.py index 6f0e502e8b..13f8559045 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -374,7 +374,7 @@ def test_close_open_pr(self, mock_patch, mock_post): } assert psync.close_open_pr(pr) - assert mock_patch.called_once_with("url_to_update_pr") + mock_patch.assert_called_once_with("url_to_update_pr") @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) @@ -397,7 +397,7 @@ def test_close_open_pr_fail(self, mock_patch, mock_post): } assert not psync.close_open_pr(pr) - assert mock_patch.called_once_with("bad_url_to_update_pr") + mock_patch.assert_called_once_with("bad_url_to_update_pr") def test_reset_target_dir(self): """Try resetting target pipeline directory""" From 7cde82f0d97c4161aa5ea144ba83a893709b2453 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 15:52:35 +0000 Subject: [PATCH 037/142] fix test-datasets --- tests/subworkflows/lint.py | 4 ++-- tests/test_modules.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 84ed6d4234..a315ea3912 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -187,12 +187,12 @@ def test_subworkflows_absent_version(self): """Test linting a nf-test module if the versions is absent in the snapshot file `""" self.subworkflow_install.install("bam_sort_stats_samtools") with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_sort_stats_samtools", "test", "main.nf.test.snap") + Path(self.pipeline_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") ) as fh: content = fh.read() new_content = content.replace("versions", "foo") with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_sort_stats_samtools", "test", "main.nf.test.snap"), "w" + Path(self.pipeline_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap"), "w" ) as fh: fh.write(new_content) diff --git a/tests/test_modules.py b/tests/test_modules.py index 5f64a233f2..26fdea41ac 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -55,8 +55,8 @@ def create_modules_repo_dummy(tmp_dir): test_snap_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") test_snap_path.touch() with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n}') - fh.write('{\n "versions": {}\n}') + fh.write('{\n "my test": {}\n') + fh.write(' "versions": {}\n}') # remove "TODO" statements from main.nf main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") From a124c1479c020ad950b50e79706313b0de549ae6 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 20 Mar 2024 16:00:39 +0000 Subject: [PATCH 038/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94cc5cb407..ee67744a69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) +- Fix incorrect assertions for called_with on mocks ([#2891](https://github.com/nf-core/tools/pull/2891)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From d29f2ba52d505a89abc2e8f93ba4496d649ce9ae Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 20 Mar 2024 16:29:41 +0000 Subject: [PATCH 039/142] Update expected assertion for mock patch calls --- tests/test_sync.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 13f8559045..b94968cd4c 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -26,7 +26,12 @@ def setUp(self): self.pipeline_dir = os.path.join(self.tmp_dir, "testpipeline") default_branch = "master" self.create_obj = nf_core.create.PipelineCreate( - "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True, default_branch=default_branch + "testing", + "test pipeline", + "tester", + outdir=self.pipeline_dir, + plain=True, + default_branch=default_branch, ) self.create_obj.init_pipeline() self.remote_path = os.path.join(self.tmp_dir, "remote_repo") @@ -374,7 +379,7 @@ def test_close_open_pr(self, mock_patch, mock_post): } assert psync.close_open_pr(pr) - mock_patch.assert_called_once_with("url_to_update_pr") + mock_patch.assert_called_once_with(url="url_to_update_pr", data='{"state": "closed"}') @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) @@ -397,7 +402,7 @@ def test_close_open_pr_fail(self, mock_patch, mock_post): } assert not psync.close_open_pr(pr) - mock_patch.assert_called_once_with("bad_url_to_update_pr") + mock_patch.assert_called_once_with(url="bad_url_to_update_pr", data='{"state": "closed"}') def test_reset_target_dir(self): """Try resetting target pipeline directory""" From 304e41fee50cb817e29568bbf4d442c8f2d6b430 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 21 Mar 2024 19:25:19 +0100 Subject: [PATCH 040/142] use pathlib --- nf_core/lint/nfcore_yml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index 64fefa50b8..aeae000e05 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -1,5 +1,5 @@ -import os import re +from pathlib import Path from nf_core import __version__ @@ -29,10 +29,10 @@ def nfcore_yml(self): ignore_configs = self.lint_config.get(".nf-core", []) try: - with open(os.path.join(self.wf_path, ".nf-core.yml")) as fh: + with open(Path(self.wf_path, ".nf-core.yml")) as fh: content = fh.read() except FileNotFoundError: - with open(os.path.join(self.wf_path, ".nf-core.yaml")) as fh: + with open(Path(self.wf_path, ".nf-core.yaml")) as fh: content = fh.read() if "repository_type" not in ignore_configs: From 585d591dea93259b9393d6f0c7a39f8a8a3b0780 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 21 Mar 2024 19:28:43 +0100 Subject: [PATCH 041/142] add type hints --- nf_core/lint/nfcore_yml.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index aeae000e05..9f45355e76 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -1,12 +1,13 @@ import re from pathlib import Path +from typing import Dict, List from nf_core import __version__ REPOSITORY_TYPES = ["pipeline", "modules"] -def nfcore_yml(self): +def nfcore_yml(self) -> Dict[str, List[str]]: """Repository ``.nf-core.yml`` tests The ``.nf-core.yml`` contains metadata for nf-core tools to correctly apply its features. @@ -20,10 +21,10 @@ def nfcore_yml(self): * Check if the nf-core version is set to the latest version. """ - passed = [] - warned = [] - failed = [] - ignored = [] + passed: List[str] = [] + warned: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] # Remove field that should be ignored according to the linting config ignore_configs = self.lint_config.get(".nf-core", []) From e3b0949d7d3bba550513fa634b5760a2a0c34556 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 21 Mar 2024 19:31:51 +0100 Subject: [PATCH 042/142] fix formatting --- nf_core/lint/nfcore_yml.py | 20 ++++++++++---------- tests/lint/nfcore_yml.py | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index 9f45355e76..f23b2f1a84 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -44,15 +44,15 @@ def nfcore_yml(self) -> Dict[str, List[str]]: repo_type = match.group(1) if repo_type not in REPOSITORY_TYPES: failed.append( - f"Repository type in .nf-core.yml is not valid. " - f"Should be one of [{', '.join(REPOSITORY_TYPES)}] but was {repo_type}" + f"Repository type in `.nf-core.yml` is not valid. " + f"Should be one of `[{', '.join(REPOSITORY_TYPES)}]` but was `{repo_type}`" ) else: - passed.append(f"Repository type in .nf-core.yml is valid: {repo_type}") + passed.append(f"Repository type in `.nf-core.yml` is valid: `{repo_type}`") else: - warned.append("Repository type not set in .nf-core.yml") + warned.append("Repository type not set in `.nf-core.yml`") else: - ignored.append(".nf-core.yml variable ignored 'repository_type'") + ignored.append("`.nf-core.yml` variable ignored 'repository_type'") if "nf_core_version" not in ignore_configs: # Check that the nf-core version is set in the .nf-core.yml @@ -62,14 +62,14 @@ def nfcore_yml(self) -> Dict[str, List[str]]: nf_core_version = match.group(1).strip('"') if nf_core_version != __version__ and "dev" not in nf_core_version: warned.append( - f"nf-core version in .nf-core.yml is not set to the latest version. " - f"Should be {__version__} but was {nf_core_version}" + f"nf-core version in `.nf-core.yml` is not set to the latest version. " + f"Should be `{__version__}` but was `{nf_core_version}`" ) else: - passed.append(f"nf-core version in .nf-core.yml is set to the latest version: {nf_core_version}") + passed.append(f"nf-core version in `.nf-core.yml` is set to the latest version: `{nf_core_version}`") else: - warned.append("nf-core version not set in .nf-core.yml") + warned.append("nf-core version not set in `.nf-core.yml`") else: - ignored.append(".nf-core.yml variable ignored 'nf_core_version'") + ignored.append("`.nf-core.yml` variable ignored 'nf_core_version'") return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py index 74d42fe229..b75115fea4 100644 --- a/tests/lint/nfcore_yml.py +++ b/tests/lint/nfcore_yml.py @@ -10,8 +10,8 @@ def test_nfcore_yml_pass(self): self.lint_obj._load() results = self.lint_obj.nfcore_yml() - assert "Repository type in .nf-core.yml is valid" in str(results["passed"]) - assert "nf-core version in .nf-core.yml is set to the latest version" in str(results["passed"]) + assert "Repository type in `.nf-core.yml` is valid" in str(results["passed"]) + assert "nf-core version in `.nf-core.yml` is set to the latest version" in str(results["passed"]) assert len(results.get("warned", [])) == 0 assert len(results.get("failed", [])) == 0 assert len(results.get("ignored", [])) == 0 @@ -29,7 +29,7 @@ def test_nfcore_yml_fail_repo_type(self): lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() results = lint_obj.nfcore_yml() - assert "Repository type in .nf-core.yml is not valid." in str(results["failed"]) + assert "Repository type in `.nf-core.yml` is not valid." in str(results["failed"]) assert len(results.get("warned", [])) == 0 assert len(results.get("passed", [])) >= 0 assert len(results.get("ignored", [])) == 0 From 6ec89d842079151f994ec42170454978fe85657d Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 21 Mar 2024 21:13:27 +0100 Subject: [PATCH 043/142] fix test --- tests/lint/nfcore_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py index b75115fea4..474ccd48fc 100644 --- a/tests/lint/nfcore_yml.py +++ b/tests/lint/nfcore_yml.py @@ -47,7 +47,7 @@ def test_nfcore_yml_fail_nfcore_version(self): lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() results = lint_obj.nfcore_yml() - assert "nf-core version in .nf-core.yml is not set to the latest version." in str(results["warned"]) + assert "nf-core version in `.nf-core.yml` is not set to the latest version." in str(results["warned"]) assert len(results.get("failed", [])) == 0 assert len(results.get("passed", [])) >= 0 assert len(results.get("ignored", [])) == 0 From 958dfa64a92e6fd69793ea0a56e28dc70b605bfa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 20:31:31 +0000 Subject: [PATCH 044/142] Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a68848237..3ab0dceba2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.3 + rev: v0.3.4 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From e9667f449d35835c0bc96e25e446d3cb29806628 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 21 Mar 2024 20:32:16 +0000 Subject: [PATCH 045/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e740d2af..cd76d3f723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 2cb50ca50c76e724fd71b69598b60ce0cbed0d3d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:13:48 +0000 Subject: [PATCH 046/142] chore(deps): update gitpod/workspace-base docker digest to fedb722 --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 3136d02e97..32eb2f1931 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:1e133e5691add6c19443672594b9f3d7d9c3372ead4c86a4490c2701dbfa32e3 +FROM gitpod/workspace-base@sha256:fedb7224944615d57661e4fd39f4873f0cb92234f55adf83a67a0fc8b4a817dd USER root From bf0aa71fd03e85cf53d4626db7790ad906233471 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:37:28 +0000 Subject: [PATCH 047/142] chore(deps): update gitpod/workspace-base docker digest to 168d78b --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 32eb2f1931..40aa998a50 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:fedb7224944615d57661e4fd39f4873f0cb92234f55adf83a67a0fc8b4a817dd +FROM gitpod/workspace-base@sha256:168d78bc249332f9437b11a1844819308ac7ec1bc6bab61bf5adf8cbe53bf8e6 USER root From f0b9a1be7f2a43895304bfe6edebad280108fbaf Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 02:24:28 +0000 Subject: [PATCH 048/142] Pin python Docker tag to 5dc6f84 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6e785f7a60..9f49001089 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim +FROM python:3.12-slim@sha256:5dc6f84b5e97bfb0c90abfb7c55f3cacc2cb6687c8f920b64a833a2219875997 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From e28d3e2280c705ce96d099c5123a75456a03be23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Mar 2024 10:36:10 +0100 Subject: [PATCH 049/142] Remove pyproject.toml from files that must not exist --- nf_core/lint/files_exist.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index ae105d8d2c..d801caf704 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -89,7 +89,6 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: lib/WorkflowPIPELINE.groovy parameters.settings.json pipeline_template.yml # saving information in .nf-core.yml - pyproject.toml Singularity @@ -198,7 +197,6 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy"), Path("parameters.settings.json"), Path("pipeline_template.yml"), # saving information in .nf-core.yml - Path("pyproject.toml"), Path("Singularity"), ] files_warn_ifexists = [Path(".travis.yml")] From 0fd48bb296382a06c62bf17ec7f2d108172af001 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Mar 2024 11:30:26 +0100 Subject: [PATCH 050/142] Remove old references to CUSTOMDUMPSOFTWAREVERSIONS --- nf_core/pipeline-template/conf/base.config | 3 --- nf_core/pipeline-template/conf/modules.config | 8 -------- 2 files changed, 11 deletions(-) diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index f73c5afaa4..9c62bf0634 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -59,7 +59,4 @@ process { errorStrategy = 'retry' maxRetries = 2 } - withName:CUSTOM_DUMPSOFTWAREVERSIONS { - cache = false - } } diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index e3ea8fa6c4..d203d2b6e6 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -22,14 +22,6 @@ process { ext.args = '--quiet' } - withName: CUSTOM_DUMPSOFTWAREVERSIONS { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - pattern: '*_versions.yml' - ] - } - withName: 'MULTIQC' { ext.args = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' } publishDir = [ From 48a23bf6a006fb3807aa1d90998bb81b679c6c2a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 26 Mar 2024 10:32:08 +0000 Subject: [PATCH 051/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e740d2af..3f589f1104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) +- Remove old references to CUSTOMDUMPSOFTWAREVERSIONS ([#2897](https://github.com/nf-core/tools/pull/2897)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 2e863d34952f9948564bd8977f5c06eba08bd270 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Mar 2024 13:12:08 +0100 Subject: [PATCH 052/142] Apply suggestions from code review --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71dfb8eff8..cc77a01831 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,6 @@ - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) - Remove old references to CUSTOMDUMPSOFTWAREVERSIONS ([#2897](https://github.com/nf-core/tools/pull/2897)) - ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] ### Template From fb2ded4bbf57de8c5c3e3f9e97c34fc7e639ed6b Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 13:06:27 +0100 Subject: [PATCH 053/142] fix type hints --- nf_core/lint/__init__.py | 296 +++++++++++++++++++-------------------- nf_core/schema.py | 6 +- 2 files changed, 152 insertions(+), 150 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index be9ac183a6..c19d107610 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -9,7 +9,7 @@ import logging import os from pathlib import Path -from typing import List, Union +from typing import List, Tuple, Union import git import rich @@ -25,6 +25,7 @@ import nf_core.subworkflows.lint import nf_core.utils from nf_core import __version__ +from nf_core.components.lint import ComponentLint from nf_core.lint_utils import console from nf_core.utils import plural_s as _s from nf_core.utils import strip_ansi_codes @@ -32,148 +33,6 @@ log = logging.getLogger(__name__) -def run_linting( - pipeline_dir, - release_mode=False, - fix=(), - key=(), - show_passed=False, - fail_ignored=False, - fail_warned=False, - sort_by="test", - md_fn=None, - json_fn=None, - hide_progress=False, -): - """Runs all nf-core linting checks on a given Nextflow pipeline project - in either `release` mode or `normal` mode (default). Returns an object - of type :class:`PipelineLint` after finished. - - Args: - pipeline_dir (str): The path to the Nextflow pipeline root directory - release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. - See :class:`PipelineLint` for more information. - - Returns: - An object of type :class:`PipelineLint` that contains all the linting results. - An object of type :class:`ComponentLint` that contains all the linting results for the modules. - An object of type :class:`ComponentLint` that contains all the linting results for the subworkflows. - """ - - # Verify that the requested tests exist - if key: - all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union( - set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)) - ) - bad_keys = [k for k in key if k not in all_tests] - if len(bad_keys) > 0: - raise AssertionError( - "Test name{} not recognised: '{}'".format( - _s(bad_keys), - "', '".join(bad_keys), - ) - ) - log.info("Only running tests: '{}'".format("', '".join(key))) - - # Check if we were given any keys, and if they match any pipeline tests - if key: - pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) - else: - # If no key is supplied, run all tests - pipeline_keys = None - - # Create the lint object - lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored, fail_warned, hide_progress) - - # Load the various pipeline configs - lint_obj._load_lint_config() - lint_obj._load_pipeline_config() - lint_obj._list_files() - - # Create the modules lint object - module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) - # Create the subworkflows lint object - try: - subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) - except LookupError: - subworkflow_lint_obj = None - - # Verify that the pipeline is correctly configured and has a modules.json file - module_lint_obj.has_valid_directory() - module_lint_obj.has_modules_file() - - # Run only the tests we want - if key: - # Select only the module lint tests - module_lint_tests = list( - set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) - ) - # Select only the subworkflow lint tests - subworkflow_lint_tests = list( - set(key).intersection( - set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) - ) - ) - else: - # If no key is supplied, run the default modules tests - module_lint_tests = ("module_changes", "module_version") - subworkflow_lint_tests = ("subworkflow_changes", "subworkflow_version") - module_lint_obj.filter_tests_by_key(module_lint_tests) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) - - # Set up files for component linting test - module_lint_obj.set_up_pipeline_files() - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.set_up_pipeline_files() - - # Run the pipeline linting tests - try: - lint_obj._lint_pipeline() - except AssertionError as e: - log.critical(f"Critical error: {e}") - log.info("Stopping tests...") - return lint_obj, module_lint_obj - - # Run the module lint tests - if len(module_lint_obj.all_local_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) - if len(module_lint_obj.all_remote_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) - # Run the subworkflows lint tests - if subworkflow_lint_obj is not None: - if len(subworkflow_lint_obj.all_local_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) - if len(subworkflow_lint_obj.all_remote_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) - - # Print the results - lint_obj._print_results(show_passed) - module_lint_obj._print_results(show_passed, sort_by=sort_by) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) - nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) - nf_core.lint_utils.print_fixes(lint_obj) - - # Save results to Markdown file - if md_fn is not None: - log.info(f"Writing lint results to {md_fn}") - markdown = lint_obj._get_results_md() - with open(md_fn, "w") as fh: - fh.write(markdown) - - # Save results to JSON file - if json_fn is not None: - lint_obj._save_json_results(json_fn) - - # Reminder about --release mode flag if we had failures - if len(lint_obj.failed) > 0: - if release_mode: - log.info("Reminder: Lint tests were run in --release mode.") - - return lint_obj, module_lint_obj, subworkflow_lint_obj - - class PipelineLint(nf_core.utils.Pipeline): """Object to hold linting information and results. @@ -409,7 +268,7 @@ def format_result(test_results): # Table of passed tests if len(self.passed) > 0 and show_passed: console.print( - rich.panel.Panel( + Panel( format_result(self.passed), title=rf"[bold][✔] {len(self.passed)} Pipeline Test{_s(self.passed)} Passed", title_align="left", @@ -421,7 +280,7 @@ def format_result(test_results): # Table of fixed tests if len(self.fixed) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.fixed), title=rf"[bold][?] {len(self.fixed)} Pipeline Test{_s(self.fixed)} Fixed", title_align="left", @@ -433,7 +292,7 @@ def format_result(test_results): # Table of ignored tests if len(self.ignored) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.ignored), title=rf"[bold][?] {len(self.ignored)} Pipeline Test{_s(self.ignored)} Ignored", title_align="left", @@ -445,7 +304,7 @@ def format_result(test_results): # Table of warning tests if len(self.warned) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.warned), title=rf"[bold][!] {len(self.warned)} Pipeline Test Warning{_s(self.warned)}", title_align="left", @@ -457,7 +316,7 @@ def format_result(test_results): # Table of failing tests if len(self.failed) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.failed), title=rf"[bold][✗] {len(self.failed)} Pipeline Test{_s(self.failed)} Failed", title_align="left", @@ -638,3 +497,144 @@ def _wrap_quotes(self, files: Union[List[str], List[Path], Path]) -> str: files = [files] bfiles = [f"`{str(f)}`" for f in files] return " or ".join(bfiles) + + +def run_linting( + pipeline_dir, + release_mode: bool = False, + fix=(), + key=(), + show_passed: bool = False, + fail_ignored: bool = False, + fail_warned: bool = False, + sort_by: str = "test", + md_fn=None, + json_fn=None, + hide_progress: bool = False, +) -> Tuple[PipelineLint, ComponentLint, ComponentLint | None]: + """Runs all nf-core linting checks on a given Nextflow pipeline project + in either `release` mode or `normal` mode (default). Returns an object + of type :class:`PipelineLint` after finished. + + Args: + pipeline_dir (str): The path to the Nextflow pipeline root directory + release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. + See :class:`PipelineLint` for more information. + + Returns: + An object of type :class:`PipelineLint` that contains all the linting results. + An object of type :class:`ComponentLint` that contains all the linting results for the modules. + An object of type :class:`ComponentLint` that contains all the linting results for the subworkflows. + """ + + # Verify that the requested tests exist + if key: + all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union( + set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)) + ) + bad_keys = [k for k in key if k not in all_tests] + if len(bad_keys) > 0: + raise AssertionError( + "Test name{} not recognised: '{}'".format( + _s(bad_keys), + "', '".join(bad_keys), + ) + ) + log.info("Only running tests: '{}'".format("', '".join(key))) + + # Check if we were given any keys, and if they match any pipeline tests + if key: + pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) + else: + # If no key is supplied, run all tests + pipeline_keys = None + + # Create the lint object + lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored, fail_warned, hide_progress) + + # Load the various pipeline configs + lint_obj._load_lint_config() + lint_obj._load_pipeline_config() + lint_obj._list_files() + + # Create the modules lint object + module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) + # Create the subworkflows lint object + try: + subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) + except LookupError: + subworkflow_lint_obj = None + + # Verify that the pipeline is correctly configured and has a modules.json file + module_lint_obj.has_valid_directory() + module_lint_obj.has_modules_file() + # Run only the tests we want + if key: + # Select only the module lint tests + module_lint_tests = list( + set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) + ) + # Select only the subworkflow lint tests + subworkflow_lint_tests = list( + set(key).intersection( + set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) + ) + ) + else: + # If no key is supplied, run the default modules tests + module_lint_tests = list(("module_changes", "module_version")) + subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version")) + module_lint_obj.filter_tests_by_key(module_lint_tests) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) + + # Set up files for component linting test + module_lint_obj.set_up_pipeline_files() + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.set_up_pipeline_files() + + # Run the pipeline linting tests + try: + lint_obj._lint_pipeline() + except AssertionError as e: + log.critical(f"Critical error: {e}") + log.info("Stopping tests...") + return lint_obj, module_lint_obj, subworkflow_lint_obj + + # Run the module lint tests + if len(module_lint_obj.all_local_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) + if len(module_lint_obj.all_remote_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) + # Run the subworkflows lint tests + if subworkflow_lint_obj is not None: + if len(subworkflow_lint_obj.all_local_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) + if len(subworkflow_lint_obj.all_remote_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) + + # Print the results + lint_obj._print_results(show_passed) + module_lint_obj._print_results(show_passed, sort_by=sort_by) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) + nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) + nf_core.lint_utils.print_fixes(lint_obj) + + # Save results to Markdown file + if md_fn is not None: + log.info(f"Writing lint results to {md_fn}") + markdown = lint_obj._get_results_md() + with open(md_fn, "w") as fh: + fh.write(markdown) + + # Save results to JSON file + if json_fn is not None: + lint_obj._save_json_results(json_fn) + + # Reminder about --release mode flag if we had failures + if len(lint_obj.failed) > 0: + if release_mode: + log.info("Reminder: Lint tests were run in --release mode.") + + return lint_obj, module_lint_obj, subworkflow_lint_obj diff --git a/nf_core/schema.py b/nf_core/schema.py index 4096c80f93..eabef74833 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -62,8 +62,10 @@ def get_schema_path(self, path, local_only=False, revision=None): # Path does not exist - assume a name of a remote workflow elif not local_only: - self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision) - self.schema_filename = Path(self.pipeline_dir, "nextflow_schema.json") + if self.pipeline_dir is not None: + self.schema_filename = Path(self.pipeline_dir, "nextflow_schema.json") + else: + self.schema_filename = None # Only looking for local paths, overwrite with None to be safe else: From 4fa15142ac03aebab4f9cb0323a2fdc3411346de Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 13:59:08 +0100 Subject: [PATCH 054/142] fix tests --- tests/test_cli.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 618ef0d290..1f7368af12 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -31,13 +31,13 @@ def test_header_outdated(mock_check_outdated, mock_nf_core_cli, capsys): class TestCli(unittest.TestCase): - """Class for testing the commandline interface""" + """Class for testing the command line interface""" def setUp(self): self.runner = CliRunner() def assemble_params(self, params): - """Assemble a dictionnary of parameters into a list of arguments for the cli + """Assemble a dictionary of parameters into a list of arguments for the cli Note: if the value of a parameter is None, it will be considered a flag. @@ -363,18 +363,23 @@ def test_lint_log_user_warning(self, mock_lint, mock_is_pipeline): def test_schema_lint(self, mock_get_schema_path): """Test nf-core schema lint defaults to nextflow_schema.json""" cmd = ["schema", "lint"] - result = self.invoke_cli(cmd) + Path("nextflow_schema.json").touch() + self.invoke_cli(cmd) mock_get_schema_path.assert_called_with("nextflow_schema.json") - assert "nextflow_schema.json" in result.output + + # clean up + Path("nextflow_schema.json").unlink() @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") def test_schema_lint_filename(self, mock_get_schema_path): """Test nf-core schema lint accepts a filename""" + Path("some_other_filename").touch() cmd = ["schema", "lint", "some_other_filename"] - result = self.invoke_cli(cmd) + self.invoke_cli(cmd) mock_get_schema_path.assert_called_with("some_other_filename") - assert "some_other_filename" in result.output - assert "nextflow_schema.json" not in result.output + + # clean up + Path("some_other_filename").unlink() @mock.patch("nf_core.create_logo.create_logo") def test_create_logo(self, mock_create_logo): From 016db6d9f3e84fb079258d7f1600e020e07d592b Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 15:21:15 +0100 Subject: [PATCH 055/142] try to fix the remote schema functionality --- nf_core/components/components_command.py | 2 +- nf_core/modules/modules_json.py | 22 ++++++++-------- nf_core/schema.py | 32 +++++++++++++++++++----- tests/test_schema.py | 7 +++--- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 8332429835..4df67639e2 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -245,7 +245,7 @@ def check_patch_paths(self, patch_path: Path, module_name: str) -> None: # Update path in modules.json if the file is in the correct format modules_json = ModulesJson(self.dir) modules_json.load() - if modules_json.has_git_url_and_modules(): + if modules_json.has_git_url_and_modules() and modules_json.modules_json is not None: modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ self.modules_repo.repo_path ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index f68c27b2d8..7d78268e92 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -6,7 +6,6 @@ import shutil import tempfile from pathlib import Path -from typing import Union import git import questionary @@ -32,7 +31,7 @@ class ModulesJson: An object for handling a 'modules.json' file in a pipeline """ - def __init__(self, pipeline_dir): + def __init__(self, pipeline_dir: str): """ Initialise the object. @@ -43,7 +42,7 @@ def __init__(self, pipeline_dir): self.modules_dir = Path(self.dir, "modules") self.subworkflows_dir = Path(self.dir, "subworkflows") self.modules_json_path = Path(self.dir, "modules.json") - self.modules_json: Union(dict, None) = None + self.modules_json = None self.pipeline_modules = None self.pipeline_subworkflows = None self.pipeline_components = None @@ -1051,17 +1050,18 @@ def get_component_branch(self, component_type, component, repo_url, install_dir) ) return branch - def dump(self, run_prettier: bool = False): + def dump(self, run_prettier: bool = False) -> None: """ Sort the modules.json, and write it to file """ - # Sort the modules.json - self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) - if run_prettier: - dump_json_with_prettier(self.modules_json_path, self.modules_json) - else: - with open(self.modules_json_path, "w") as fh: - json.dump(self.modules_json, fh, indent=4) + if self.modules_json is not None: + # Sort the modules.json + self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) + if run_prettier: + dump_json_with_prettier(self.modules_json_path, self.modules_json) + else: + with open(self.modules_json_path, "w") as fh: + json.dump(self.modules_json, fh, indent=4) def resolve_missing_installation(self, missing_installation, component_type): missing_but_in_mod_json = [ diff --git a/nf_core/schema.py b/nf_core/schema.py index eabef74833..a4b635c819 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -6,6 +6,7 @@ import tempfile import webbrowser from pathlib import Path +from typing import Union import jinja2 import jsonschema @@ -46,11 +47,14 @@ def __init__(self): self.web_schema_build_web_url = None self.web_schema_build_api_url = None - def get_schema_path(self, path, local_only=False, revision=None): + def get_schema_path( + self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None + ) -> None: """Given a pipeline name, directory, or path, set self.schema_filename""" path = Path(path) # Supplied path exists - assume a local pipeline directory or schema if path.exists(): + log.debug(f"Path exists: {path}. Assuming local pipeline directory or schema") if revision is not None: log.warning(f"Local workflow supplied, ignoring revision '{revision}'") if path.is_dir(): @@ -62,17 +66,20 @@ def get_schema_path(self, path, local_only=False, revision=None): # Path does not exist - assume a name of a remote workflow elif not local_only: - if self.pipeline_dir is not None: - self.schema_filename = Path(self.pipeline_dir, "nextflow_schema.json") + self.schema_filename = path / "nextflow_schema.json" + if revision is not None: + self.load_remote_schema( + f"https://raw.githubusercontent.com/nf-core/{path}/{revision}/nextflow_schema.json" + ) else: - self.schema_filename = None + self.load_remote_schema(f"https://raw.githubusercontent.com/nf-core/{path}/master/nextflow_schema.json") # Only looking for local paths, overwrite with None to be safe else: self.schema_filename = None # Check that the schema file exists - if self.schema_filename is None or not Path(self.schema_filename).exists(): + if self.schema_filename is None or not Path(self.schema_filename).exists() and local_only: error = f"Could not find pipeline schema for '{path}': {self.schema_filename}" log.error(error) raise AssertionError(error) @@ -105,7 +112,7 @@ def load_lint_schema(self): def load_schema(self): """Load a pipeline schema from a file""" - if self.schema_filename is None: + if self.schema_filename is None or not Path(self.schema_filename).exists(): raise AssertionError("Pipeline schema filename could not be found.") with open(self.schema_filename) as fh: @@ -114,6 +121,19 @@ def load_schema(self): self.schema_params = {} log.debug(f"JSON file loaded: {self.schema_filename}") + def load_remote_schema(self, url): + """Load a pipeline schema from a remote URL""" + import requests + + response = requests.get(url) + if response.status_code != 200: + raise AssertionError(f"Could not load schema from {url}") + self.schema = response.json() + self.schema_filename = url + self.schema_defaults = {} + self.schema_params = {} + log.debug(f"JSON file loaded: {self.schema_filename}") + def sanitise_param_default(self, param): """ Given a param, ensure that the default value is the correct variable type diff --git a/tests/test_schema.py b/tests/test_schema.py index 29f4921985..5c28ee92bf 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -5,6 +5,7 @@ import shutil import tempfile import unittest +from pathlib import Path from unittest import mock import pytest @@ -46,7 +47,7 @@ def test_load_lint_schema(self): def test_load_lint_schema_nofile(self): """Check that linting raises properly if a non-existant file is given""" - with pytest.raises(RuntimeError): + with pytest.raises(AssertionError): self.schema_obj.get_schema_path("fake_file") def test_load_lint_schema_notjson(self): @@ -314,9 +315,9 @@ def test_build_schema_from_scratch(self, tmp_dir): Pretty much a copy of test_launch.py test_make_pipeline_schema """ - test_pipeline_dir = os.path.join(tmp_dir, "wf") + test_pipeline_dir = Path(tmp_dir, "wf") shutil.copytree(self.template_dir, test_pipeline_dir) - os.remove(os.path.join(test_pipeline_dir, "nextflow_schema.json")) + Path(test_pipeline_dir, "nextflow_schema.json").unlink() self.schema_obj.build_schema(test_pipeline_dir, True, False, None) From b69f328c2d441e64aa5ded97a8bd850741e1720d Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 15:24:43 +0100 Subject: [PATCH 056/142] add isolated filesystem to cli tests --- tests/test_cli.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 1f7368af12..913a4aac1d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -363,23 +363,21 @@ def test_lint_log_user_warning(self, mock_lint, mock_is_pipeline): def test_schema_lint(self, mock_get_schema_path): """Test nf-core schema lint defaults to nextflow_schema.json""" cmd = ["schema", "lint"] - Path("nextflow_schema.json").touch() - self.invoke_cli(cmd) - mock_get_schema_path.assert_called_with("nextflow_schema.json") - - # clean up - Path("nextflow_schema.json").unlink() + with self.runner.isolated_filesystem(): + with open("nextflow_schema.json", "w") as f: + f.write("{}") + self.invoke_cli(cmd) + mock_get_schema_path.assert_called_with("nextflow_schema.json") @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") def test_schema_lint_filename(self, mock_get_schema_path): """Test nf-core schema lint accepts a filename""" - Path("some_other_filename").touch() cmd = ["schema", "lint", "some_other_filename"] - self.invoke_cli(cmd) - mock_get_schema_path.assert_called_with("some_other_filename") - - # clean up - Path("some_other_filename").unlink() + with self.runner.isolated_filesystem(): + with open("some_other_filename", "w") as f: + f.write("{}") + self.invoke_cli(cmd) + mock_get_schema_path.assert_called_with("some_other_filename") @mock.patch("nf_core.create_logo.create_logo") def test_create_logo(self, mock_create_logo): From 738e4e0db6a07b9c59ed6a1f0853af6e4f326eb7 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 15:31:40 +0100 Subject: [PATCH 057/142] use correct types --- nf_core/lint/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 1dda6a2078..2a8576d5fb 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -513,7 +513,7 @@ def run_linting( md_fn=None, json_fn=None, hide_progress: bool = False, -) -> Tuple[PipelineLint, ComponentLint, ComponentLint | None]: +) -> Tuple[PipelineLint, ComponentLint, Union[ComponentLint, None]]: """Runs all nf-core linting checks on a given Nextflow pipeline project in either `release` mode or `normal` mode (default). Returns an object of type :class:`PipelineLint` after finished. From 029617703a05429f61411b2e92cc9acdebc1e58c Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 16:58:29 +0100 Subject: [PATCH 058/142] use base class for moduels and base config --- nf_core/lint/__init__.py | 4 ++ nf_core/lint/base_config.py | 14 +++++++ nf_core/lint/modules_config.py | 14 +++++++ nf_core/lint_utils.py | 68 ++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 nf_core/lint/base_config.py create mode 100644 nf_core/lint/modules_config.py diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index be9ac183a6..92a699ac1c 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -199,9 +199,11 @@ class PipelineLint(nf_core.utils.Pipeline): from .actions_schema_validation import ( # type: ignore[misc] actions_schema_validation, ) + from .base_config import base_config # type: ignore[misc] from .files_exist import files_exist # type: ignore[misc] from .files_unchanged import files_unchanged # type: ignore[misc] from .merge_markers import merge_markers # type: ignore[misc] + from .modules_config import modules_config # type: ignore[misc] from .modules_json import modules_json # type: ignore[misc] from .modules_structure import modules_structure # type: ignore[misc] from .multiqc_config import multiqc_config # type: ignore[misc] @@ -264,6 +266,8 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "base_config", + "modules_config", ] + (["version_consistency"] if release_mode else []) def _load(self): diff --git a/nf_core/lint/base_config.py b/nf_core/lint/base_config.py new file mode 100644 index 0000000000..cd1385b1ca --- /dev/null +++ b/nf_core/lint/base_config.py @@ -0,0 +1,14 @@ +from pathlib import Path +from typing import Dict, List + +from nf_core.lint_utils import LintFile + + +def base_config(self) -> Dict[str, List[str]]: + """Make sure the conf/base.config file follows the nf-core template, especially removed sections.""" + + result = LintFile(self.wf_path, self.lint_config).lint_file( + "base_config", Path("conf", "base.config"), ["withName:CUSTOM_DUMPSOFTWAREVERSIONS"] + ) + + return result diff --git a/nf_core/lint/modules_config.py b/nf_core/lint/modules_config.py new file mode 100644 index 0000000000..53c58766ea --- /dev/null +++ b/nf_core/lint/modules_config.py @@ -0,0 +1,14 @@ +from pathlib import Path +from typing import Dict, List + +from nf_core.lint_utils import LintFile + + +def modules_config(self) -> Dict[str, List[str]]: + """Make sure the conf/modules.config file follows the nf-core template, especially removed sections.""" + + result = LintFile(self.wf_path, self.lint_config).lint_file( + "modules_config", Path("conf", "modules.config"), ["withName:CUSTOM_DUMPSOFTWAREVERSIONS"] + ) + + return result diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 6eca6522d4..e099fcabf3 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -2,6 +2,7 @@ import logging import subprocess from pathlib import Path +from typing import Dict, List, Tuple import rich from rich.console import Console @@ -16,6 +17,42 @@ console = Console(force_terminal=nf_core.utils.rich_force_colors()) +class LintFile: + def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): + self.wf_path = wf_path + self.lint_config = lint_config + + def lint_file(self, lint_name: str, file_path: Path, removed_sections: List[str]) -> Dict[str, List[str]]: + """Lint a file and add the result to the passed or failed list.""" + + fn = Path(self.wf_path, file_path) + passed: List[str] = [] + failed: List[str] = [] + + ignore_configs = self.lint_config.get(lint_name, []) + + # Return a failed status if we can't find the file + if not fn.is_file(): + return {"ignored": [f"`${file_path}` not found"]} + + try: + with open(fn) as fh: + modules_config = fh.read() + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]} + + # check if removed sections are absent + + for section in removed_sections: + if section in modules_config and section not in ignore_configs: + failed.append(f"`${file_path}` contains `{section}`") + return {"passed": passed, "failed": failed} + else: + passed.append(f"`${file_path}` does not contain `{section}`") + + return {"passed": passed, "failed": failed} + + def print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj): """Print a joint summary of the general pipe lint tests and the module and subworkflow lint tests""" swf_passed = 0 @@ -101,3 +138,34 @@ def dump_json_with_prettier(file_name, file_content): with open(file_name, "w") as fh: json.dump(file_content, fh, indent=4) run_prettier_on_file(file_name) + + +def parse_config_file(self, lint_name: str, file_path: Path) -> Tuple[dict, dict]: + """Parse different kind of config files and return a dict.""" + + # Remove field that should be ignored according to the linting config + ignore_configs = self.lint_config.get(lint_name, []) + + fn = Path(self.wf_path, file_path) + + # Return a failed status if we can't find the file + if not fn.is_file(): + return {"ignored": [f"`${file_path}` not found"]}, ignore_configs + + try: + if fn.suffix == ".json": + import json + + with open(fn) as fh: + config = json.load(fh) + return config, ignore_configs + elif fn.suffix == ".yml" or fn.suffix == ".yaml": + import yaml + + with open(fn) as fh: + config = yaml.safe_load(fh) + return config, ignore_configs + else: + return {"failed": [f"Could not parse file: {fn}, unknown file type"]}, ignore_configs + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]}, ignore_configs From e9de4d9fd9cbc0268e9a48bd058fc09e89cccbba Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 26 Mar 2024 16:25:36 +0000 Subject: [PATCH 059/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e740d2af..adceeca6e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) +- Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From e9d0895144d685e296eb5f5b56bedc6274fb876f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 18:22:46 +0100 Subject: [PATCH 060/142] rewrite everything in a different way: Part III --- .../_src/pipeline_lint_tests/base_config.md | 5 ++ .../pipeline_lint_tests/modules_config.md | 5 ++ nf_core/lint/__init__.py | 3 +- nf_core/lint/base_config.py | 14 ---- nf_core/lint/configs.py | 64 +++++++++++++++++++ nf_core/lint/modules_config.py | 14 ---- nf_core/lint_utils.py | 38 +---------- nf_core/utils.py | 25 ++++---- 8 files changed, 89 insertions(+), 79 deletions(-) create mode 100644 docs/api/_src/pipeline_lint_tests/base_config.md create mode 100644 docs/api/_src/pipeline_lint_tests/modules_config.md delete mode 100644 nf_core/lint/base_config.py create mode 100644 nf_core/lint/configs.py delete mode 100644 nf_core/lint/modules_config.py diff --git a/docs/api/_src/pipeline_lint_tests/base_config.md b/docs/api/_src/pipeline_lint_tests/base_config.md new file mode 100644 index 0000000000..4a56ef9789 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/base_config.md @@ -0,0 +1,5 @@ +# base_config + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.base_config +``` diff --git a/docs/api/_src/pipeline_lint_tests/modules_config.md b/docs/api/_src/pipeline_lint_tests/modules_config.md new file mode 100644 index 0000000000..2a4f51c5a4 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/modules_config.md @@ -0,0 +1,5 @@ +# modules_config + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.modules_config +``` diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 92a699ac1c..aaa46d0424 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -199,11 +199,10 @@ class PipelineLint(nf_core.utils.Pipeline): from .actions_schema_validation import ( # type: ignore[misc] actions_schema_validation, ) - from .base_config import base_config # type: ignore[misc] + from .configs import base_config, modules_config # type: ignore[misc] from .files_exist import files_exist # type: ignore[misc] from .files_unchanged import files_unchanged # type: ignore[misc] from .merge_markers import merge_markers # type: ignore[misc] - from .modules_config import modules_config # type: ignore[misc] from .modules_json import modules_json # type: ignore[misc] from .modules_structure import modules_structure # type: ignore[misc] from .multiqc_config import multiqc_config # type: ignore[misc] diff --git a/nf_core/lint/base_config.py b/nf_core/lint/base_config.py deleted file mode 100644 index cd1385b1ca..0000000000 --- a/nf_core/lint/base_config.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path -from typing import Dict, List - -from nf_core.lint_utils import LintFile - - -def base_config(self) -> Dict[str, List[str]]: - """Make sure the conf/base.config file follows the nf-core template, especially removed sections.""" - - result = LintFile(self.wf_path, self.lint_config).lint_file( - "base_config", Path("conf", "base.config"), ["withName:CUSTOM_DUMPSOFTWAREVERSIONS"] - ) - - return result diff --git a/nf_core/lint/configs.py b/nf_core/lint/configs.py new file mode 100644 index 0000000000..5cae126bb9 --- /dev/null +++ b/nf_core/lint/configs.py @@ -0,0 +1,64 @@ +from pathlib import Path +from typing import Dict, List + + +class LintConfig: + def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): + self.wf_path = wf_path + self.lint_config = lint_config + + def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: + """Lint a file and add the result to the passed or failed list.""" + + fn = Path(self.wf_path, file_path) + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + + ignore_configs = self.lint_config.get(lint_name, []) + + # Return a failed status if we can't find the file + if not fn.is_file(): + return {"failed": [f"`${file_path}` not found"]} + + try: + with open(fn) as fh: + config = fh.read() + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]} + + # find sections with a withName: prefix + sections = [line.split(":")[1].strip().split(" ")[0] for line in config.split("\n") if "withName:" in line] + + # find all .nf files in the workflow directory + nf_files = list(Path(self.wf_path).rglob("*.nf")) + + # check if withName sections are present in config, but not in workflow files + for section in sections: + if section not in ignore_configs: + if not any(section in nf_file.read_text() for nf_file in nf_files): + failed.append( + f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the following workflow files: `{nf_files}`." + ) + else: + passed.append(f"both `{file_path}` and `{[str(f) for f in nf_files]} contain `{section}`.") + else: + ignored.append(f"``{section}` is ignored") + + return {"passed": passed, "failed": failed} + + +def modules_config(self) -> Dict[str, List[str]]: + """Make sure the conf/modules.config file follows the nf-core template, especially removed sections.""" + + result = LintConfig(self.wf_path, self.lint_config).lint_file("modules_config", Path("conf", "modules.config")) + + return result + + +def base_config(self) -> Dict[str, List[str]]: + """Make sure the conf/base.config file follows the nf-core template, especially removed sections.""" + + result = LintConfig(self.wf_path, self.lint_config).lint_file("base_config", Path("conf", "base.config")) + + return result diff --git a/nf_core/lint/modules_config.py b/nf_core/lint/modules_config.py deleted file mode 100644 index 53c58766ea..0000000000 --- a/nf_core/lint/modules_config.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path -from typing import Dict, List - -from nf_core.lint_utils import LintFile - - -def modules_config(self) -> Dict[str, List[str]]: - """Make sure the conf/modules.config file follows the nf-core template, especially removed sections.""" - - result = LintFile(self.wf_path, self.lint_config).lint_file( - "modules_config", Path("conf", "modules.config"), ["withName:CUSTOM_DUMPSOFTWAREVERSIONS"] - ) - - return result diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index e099fcabf3..27fb36568f 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -2,7 +2,7 @@ import logging import subprocess from pathlib import Path -from typing import Dict, List, Tuple +from typing import Tuple import rich from rich.console import Console @@ -17,42 +17,6 @@ console = Console(force_terminal=nf_core.utils.rich_force_colors()) -class LintFile: - def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): - self.wf_path = wf_path - self.lint_config = lint_config - - def lint_file(self, lint_name: str, file_path: Path, removed_sections: List[str]) -> Dict[str, List[str]]: - """Lint a file and add the result to the passed or failed list.""" - - fn = Path(self.wf_path, file_path) - passed: List[str] = [] - failed: List[str] = [] - - ignore_configs = self.lint_config.get(lint_name, []) - - # Return a failed status if we can't find the file - if not fn.is_file(): - return {"ignored": [f"`${file_path}` not found"]} - - try: - with open(fn) as fh: - modules_config = fh.read() - except Exception as e: - return {"failed": [f"Could not parse file: {fn}, {e}"]} - - # check if removed sections are absent - - for section in removed_sections: - if section in modules_config and section not in ignore_configs: - failed.append(f"`${file_path}` contains `{section}`") - return {"passed": passed, "failed": failed} - else: - passed.append(f"`${file_path}` does not contain `{section}`") - - return {"passed": passed, "failed": failed} - - def print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj): """Print a joint summary of the general pipe lint tests and the module and subworkflow lint tests""" swf_passed = 0 diff --git a/nf_core/utils.py b/nf_core/utils.py index 4271e971a1..83136c114e 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -216,7 +216,7 @@ def is_pipeline_directory(wf_path): raise UserWarning(f"'{wf_path}' is not a pipeline - '{fn}' is missing") -def fetch_wf_config(wf_path, cache_config=True): +def fetch_wf_config(wf_path: str, cache_config: bool = True) -> dict: """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. @@ -236,13 +236,13 @@ def fetch_wf_config(wf_path, cache_config=True): cache_path = None # Nextflow home directory - use env var if set, or default to ~/.nextflow - nxf_home = os.environ.get("NXF_HOME", os.path.join(os.getenv("HOME"), ".nextflow")) + nxf_home = Path(os.environ.get("NXF_HOME", Path(os.getenv("HOME") or "", ".nextflow"))) # Build a cache directory if we can - if os.path.isdir(nxf_home): - cache_basedir = os.path.join(nxf_home, "nf-core") - if not os.path.isdir(cache_basedir): - os.mkdir(cache_basedir) + if nxf_home.is_dir(): + cache_basedir = Path(nxf_home, "nf-core") + if not cache_basedir.is_dir(): + cache_basedir.mkdir(parents=True, exist_ok=True) # If we're given a workflow object with a commit, see if we have a cached copy cache_fn = None @@ -250,7 +250,7 @@ def fetch_wf_config(wf_path, cache_config=True): concat_hash = "" for fn in ["nextflow.config", "main.nf"]: try: - with open(os.path.join(wf_path, fn), "rb") as fh: + with open(Path(wf_path, fn), "rb") as fh: concat_hash += hashlib.sha256(fh.read()).hexdigest() except FileNotFoundError: pass @@ -260,8 +260,8 @@ def fetch_wf_config(wf_path, cache_config=True): cache_fn = f"wf-config-cache-{bighash[:25]}.json" if cache_basedir and cache_fn: - cache_path = os.path.join(cache_basedir, cache_fn) - if os.path.isfile(cache_path) and cache_config is True: + cache_path = Path(cache_basedir, cache_fn) + if cache_path.is_file() and cache_config is True: log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path) as fh: try: @@ -286,10 +286,11 @@ def fetch_wf_config(wf_path, cache_config=True): # Scrape main.nf for additional parameter declarations # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. try: - main_nf = os.path.join(wf_path, "main.nf") - with open(main_nf) as fh: + main_nf = Path(wf_path, "main.nf") + with open(main_nf, "rb") as fh: for line in fh: - match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line) + line_str = line.decode("utf-8") + match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line_str) if match: config[match.group(1)] = "null" except FileNotFoundError as e: From f7abb401beada50acb64f3749e1c47d1c51d04f6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 18:23:46 +0100 Subject: [PATCH 061/142] add tests for linting tests --- tests/lint/configs.py | 43 +++++++++++++++++++++++++++++++++++++++++++ tests/test_lint.py | 5 +++++ 2 files changed, 48 insertions(+) create mode 100644 tests/lint/configs.py diff --git a/tests/lint/configs.py b/tests/lint/configs.py new file mode 100644 index 0000000000..a4bbe0147c --- /dev/null +++ b/tests/lint/configs.py @@ -0,0 +1,43 @@ +from pathlib import Path + +import nf_core.create +import nf_core.lint + + +def test_withname_in_modules_config(self): + """Tests finding withName in modules.config passes linting.""" + + new_pipeline = self._make_pipeline_copy() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["failed"]) == 0 + assert any(["contain `FASTQC`" in passed for passed in result["passed"]]) + + +def test_superfluous_withname_in_modules_config_fails(self): + """Tests finding withName in modules.config fails linting.""" + new_pipeline = self._make_pipeline_copy() + # Add withName to modules.config + modules_config = Path(new_pipeline) / "conf" / "modules.config" + with open(modules_config, "a") as f: + f.write("withName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["failed"]) == 1 + assert result["failed"][0].startswith("`conf/modules.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") + + +def test_superfluous_withname_in_base_config_fails(self): + """Tests finding withName in base.config fails linting.""" + new_pipeline = self._make_pipeline_copy() + # Add withName to base.config + base_config = Path(new_pipeline) / "conf" / "base.config" + with open(base_config, "a") as f: + f.write("withName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.base_config() + assert len(result["failed"]) == 1 + assert result["failed"][0].startswith("`conf/base.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") diff --git a/tests/test_lint.py b/tests/test_lint.py index d10cef37e4..290b9b73c7 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -198,6 +198,11 @@ def test_sphinx_md_files(self): test_actions_schema_validation_missing_jobs, test_actions_schema_validation_missing_on, ) + from .lint.configs import ( # type: ignore[misc] + test_superfluous_withname_in_base_config_fails, + test_superfluous_withname_in_modules_config_fails, + test_withname_in_modules_config, + ) from .lint.files_exist import ( # type: ignore[misc] test_files_exist_depreciated_file, test_files_exist_fail_conditional, From a7e5593a62ad3e93f28ddc9adfc0a84e65cebf51 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 18:50:34 +0100 Subject: [PATCH 062/142] revert remote changes --- nf_core/schema.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index a4b635c819..08c8403d85 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -66,13 +66,8 @@ def get_schema_path( # Path does not exist - assume a name of a remote workflow elif not local_only: - self.schema_filename = path / "nextflow_schema.json" - if revision is not None: - self.load_remote_schema( - f"https://raw.githubusercontent.com/nf-core/{path}/{revision}/nextflow_schema.json" - ) - else: - self.load_remote_schema(f"https://raw.githubusercontent.com/nf-core/{path}/master/nextflow_schema.json") + self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision) + self.schema_filename = Path(self.pipeline_dir or "", "nextflow_schema.json") # Only looking for local paths, overwrite with None to be safe else: @@ -121,19 +116,6 @@ def load_schema(self): self.schema_params = {} log.debug(f"JSON file loaded: {self.schema_filename}") - def load_remote_schema(self, url): - """Load a pipeline schema from a remote URL""" - import requests - - response = requests.get(url) - if response.status_code != 200: - raise AssertionError(f"Could not load schema from {url}") - self.schema = response.json() - self.schema_filename = url - self.schema_defaults = {} - self.schema_params = {} - log.debug(f"JSON file loaded: {self.schema_filename}") - def sanitise_param_default(self, param): """ Given a param, ensure that the default value is the correct variable type From 3fa8131a30cb60d727d4dd6a9b56454fe10d1e13 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 19:02:08 +0100 Subject: [PATCH 063/142] fix tests --- nf_core/schema.py | 4 +++- tests/test_schema.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 08c8403d85..a4e8ad7cdc 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -68,7 +68,9 @@ def get_schema_path( elif not local_only: self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision) self.schema_filename = Path(self.pipeline_dir or "", "nextflow_schema.json") - + # check if the schema file exists + if not self.schema_filename.exists(): + self.schema_filename = None # Only looking for local paths, overwrite with None to be safe else: self.schema_filename = None diff --git a/tests/test_schema.py b/tests/test_schema.py index 5c28ee92bf..b9cb108fae 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -47,7 +47,7 @@ def test_load_lint_schema(self): def test_load_lint_schema_nofile(self): """Check that linting raises properly if a non-existant file is given""" - with pytest.raises(AssertionError): + with pytest.raises(RuntimeError): self.schema_obj.get_schema_path("fake_file") def test_load_lint_schema_notjson(self): From 0248740c255afbf75b0b7e6009d1afb0c83e0b7f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 00:56:45 +0000 Subject: [PATCH 064/142] chore(deps): update github actions --- .github/workflows/changelog.yml | 4 ++-- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/deploy-pypi.yml | 2 +- .github/workflows/fix-linting.yml | 2 +- .github/workflows/lint-code.yml | 2 +- .github/workflows/pytest.yml | 6 +++--- .github/workflows/rich-codex.yml | 2 +- .github/workflows/sync.yml | 2 +- .github/workflows/update_components_template.yml | 2 +- .../.github/workflows/download_pipeline.yml | 2 +- nf_core/pipeline-template/.github/workflows/fix-linting.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 4 ++-- .../.github/workflows/release-announcements.yml | 2 +- 15 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 50d835fd5c..bed104e2d9 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -36,7 +36,7 @@ jobs: fi gh pr checkout $PR_NUMBER - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" @@ -65,7 +65,7 @@ jobs: echo "File changed: ${{ env.changed }}" - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" cache: "pip" diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 3e93918ed1..996bdf2799 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -53,7 +53,7 @@ jobs: # Set up nf-core/tools - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" cache: pip diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 988e5bb5b0..4ef448e5ec 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -65,7 +65,7 @@ jobs: name: Check out source-code repository - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 3b7c02ca86..952181c044 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -52,7 +52,7 @@ jobs: name: Check out source-code repository - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index ca698dad1f..ec94ddb88f 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -17,7 +17,7 @@ jobs: name: Check out source-code repository - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 3e6c4d4ef6..b3f73c5a8c 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -32,7 +32,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} # Install and run pre-commit - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 53151a8989..51a543c143 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" cache: "pip" diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b03aab7cdf..93a44aea5e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -91,7 +91,7 @@ jobs: name: Check out source-code repository - name: Set up Python ${{ needs.setup.outputs.python-version }} - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: ${{ needs.setup.outputs.python-version }} cache: "pip" @@ -160,7 +160,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 env: AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/ with: @@ -184,7 +184,7 @@ jobs: coverage report coverage xml - - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4 + - uses: codecov/codecov-action@c16abc29c95fcf9174b58eb7e1abf4c866893bc8 # v4 with: files: coverage.xml env: diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 13fd71a883..fc8af8a7bc 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -8,7 +8,7 @@ jobs: - name: Check out the repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: 3.x cache: pip diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 9eeb32ccf3..2adb970f64 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -57,7 +57,7 @@ jobs: fetch-depth: "0" - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index 0fa3adaf20..c5413cd557 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.x" diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 3b791980f2..ebea16c5cb 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -35,7 +35,7 @@ jobs: - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" architecture: "x64" diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index e976b9f09c..bc15ce42e8 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -32,7 +32,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} # Install and run pre-commit - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 90cca8c2ce..a8208b31ac 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" cache: "pip" @@ -37,7 +37,7 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@v2 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" architecture: "x64" diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index 2c57060257..8fee061fdd 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.10" - name: Install dependencies From fe4383a61786756c6ac4d92f58322961206a7331 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 28 Mar 2024 02:23:07 +0000 Subject: [PATCH 065/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0a91902d3..1a3eb103bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) - Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) +- Update GitHub Actions ([#2902](https://github.com/nf-core/tools/pull/2902)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 97da643b4d968809a8853b079f44d39e91d52f10 Mon Sep 17 00:00:00 2001 From: Alex Hermida Date: Tue, 19 Mar 2024 22:18:52 +0100 Subject: [PATCH 066/142] Get immediate path parent name --- nf_core/schema.py | 4 +++- tests/test_schema.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index a4e8ad7cdc..4f5acfa0af 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -587,10 +587,12 @@ def make_skeleton_schema(self): loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) schema_template = env.get_template("nextflow_schema.json") + template_vars = { - "name": self.pipeline_manifest.get("name", str(Path(self.schema_filename).parent)).strip("'"), + "name": self.pipeline_manifest.get("name", Path(self.schema_filename).parent.name).strip("'"), "description": self.pipeline_manifest.get("description", "").strip("'"), } + self.schema = json.loads(schema_template.render(template_vars)) self.get_schema_defaults() diff --git a/tests/test_schema.py b/tests/test_schema.py index b9cb108fae..a05fdf6b92 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -218,6 +218,13 @@ def test_make_skeleton_schema(self): self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) + def test_make_skeleton_schema_absent_name(self): + """Test making a new schema skeleton""" + self.schema_obj.schema_filename = self.template_schema + self.schema_obj.pipeline_manifest["description"] = "Test pipeline" + self.schema_obj.make_skeleton_schema() + self.schema_obj.validate_schema(self.schema_obj.schema) + def test_get_wf_params(self): """Test getting the workflow parameters from a pipeline""" self.schema_obj.schema_filename = self.template_schema From b9c4746ff8bb648c4e84f6f38bd868678df7ade0 Mon Sep 17 00:00:00 2001 From: Alex Hermida Date: Tue, 19 Mar 2024 22:34:08 +0100 Subject: [PATCH 067/142] Update Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac60a404e4..79d29ca32d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ - Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) - Update GitHub Actions ([#2902](https://github.com/nf-core/tools/pull/2902)) +- Get immediate parent path name for schema creation ([#2886](https://github.com/nf-core/tools/pull/2886)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 31240de0d2c826c120813a9e84237ba7a77a4d83 Mon Sep 17 00:00:00 2001 From: Alex Hermida Date: Wed, 20 Mar 2024 14:56:19 +0100 Subject: [PATCH 068/142] Update tests --- tests/test_schema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_schema.py b/tests/test_schema.py index a05fdf6b92..b69b61762f 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -217,6 +217,7 @@ def test_make_skeleton_schema(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) + assert self.schema_obj.schema['title'] == "nf-core/test pipeline parameters" def test_make_skeleton_schema_absent_name(self): """Test making a new schema skeleton""" @@ -224,6 +225,7 @@ def test_make_skeleton_schema_absent_name(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) + assert self.schema_obj.schema['title'] == "wf pipeline parameters" def test_get_wf_params(self): """Test getting the workflow parameters from a pipeline""" From 2975f6af13bd4476739aadb499d6c707ddefa02a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 20 Mar 2024 14:23:08 +0000 Subject: [PATCH 069/142] [automated] Fix code linting --- tests/test_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index b69b61762f..e0921908d4 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -217,7 +217,7 @@ def test_make_skeleton_schema(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) - assert self.schema_obj.schema['title'] == "nf-core/test pipeline parameters" + assert self.schema_obj.schema["title"] == "nf-core/test pipeline parameters" def test_make_skeleton_schema_absent_name(self): """Test making a new schema skeleton""" @@ -225,7 +225,7 @@ def test_make_skeleton_schema_absent_name(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) - assert self.schema_obj.schema['title'] == "wf pipeline parameters" + assert self.schema_obj.schema["title"] == "wf pipeline parameters" def test_get_wf_params(self): """Test getting the workflow parameters from a pipeline""" From 654a205590d8660af66e1390c494359e1b8aeec7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 18:49:57 +0000 Subject: [PATCH 070/142] Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ab0dceba2..915bbc345d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.4 + rev: v0.3.5 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 5d9cb5af53983baa89f00eacd38d3999216980a5 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 1 Apr 2024 18:50:51 +0000 Subject: [PATCH 071/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac60a404e4..cbfef3bcc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ - Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) - Update GitHub Actions ([#2902](https://github.com/nf-core/tools/pull/2902)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 ([#2903](https://github.com/nf-core/tools/pull/2903)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 9af060d6b37499be9777941990f9b7b19f82d9a6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 2 Apr 2024 10:23:23 +0200 Subject: [PATCH 072/142] update according to review comments --- nf_core/lint/configs.py | 48 +++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/nf_core/lint/configs.py b/nf_core/lint/configs.py index 5cae126bb9..ef8be27e87 100644 --- a/nf_core/lint/configs.py +++ b/nf_core/lint/configs.py @@ -1,6 +1,10 @@ +import logging +import re from pathlib import Path from typing import Dict, List +log = logging.getLogger(__name__) + class LintConfig: def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): @@ -19,7 +23,10 @@ def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: # Return a failed status if we can't find the file if not fn.is_file(): - return {"failed": [f"`${file_path}` not found"]} + if ignore_configs: + return {"ignored": [f"`{file_path}` not found, but it is ignored."]} + else: + return {"failed": [f"`${file_path}` not found"]} try: with open(fn) as fh: @@ -28,14 +35,15 @@ def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: return {"failed": [f"Could not parse file: {fn}, {e}"]} # find sections with a withName: prefix - sections = [line.split(":")[1].strip().split(" ")[0] for line in config.split("\n") if "withName:" in line] + sections = re.findall(r"['\"](.*)['\"]", config) # find all .nf files in the workflow directory nf_files = list(Path(self.wf_path).rglob("*.nf")) + log.debug(f"found nf_files: {nf_files}") # check if withName sections are present in config, but not in workflow files for section in sections: - if section not in ignore_configs: + if section not in ignore_configs or section.lower() not in ignore_configs: if not any(section in nf_file.read_text() for nf_file in nf_files): failed.append( f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the following workflow files: `{nf_files}`." @@ -45,11 +53,29 @@ def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: else: ignored.append(f"``{section}` is ignored") - return {"passed": passed, "failed": failed} + return {"passed": passed, "failed": failed, "ignored": ignored} def modules_config(self) -> Dict[str, List[str]]: - """Make sure the conf/modules.config file follows the nf-core template, especially removed sections.""" + """Make sure the conf/modules.config file follows the nf-core template, especially removed sections. + + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: + + .. code-block:: yaml + + lint: + modules_config: False + + To disable this test only for specific modules, you can specify a list of module names. + + .. code-block:: yaml + + lint: + modules_config: + - fastqc + + """ result = LintConfig(self.wf_path, self.lint_config).lint_file("modules_config", Path("conf", "modules.config")) @@ -57,7 +83,17 @@ def modules_config(self) -> Dict[str, List[str]]: def base_config(self) -> Dict[str, List[str]]: - """Make sure the conf/base.config file follows the nf-core template, especially removed sections.""" + """Make sure the conf/base.config file follows the nf-core template, especially removed sections. + + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: + + .. code-block:: yaml + + lint: + base_config: False + + """ result = LintConfig(self.wf_path, self.lint_config).lint_file("base_config", Path("conf", "base.config")) From 6c717dcb2189353a7e25e63cbfd9740c2274407e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 2 Apr 2024 17:33:15 +0200 Subject: [PATCH 073/142] combine file check and add tests --- nf_core/lint/configs.py | 67 +++++------ nf_core/lint/multiqc_config.py | 196 +++++++++++++++++---------------- nf_core/lint_utils.py | 52 +++++---- nf_core/utils.py | 9 +- tests/lint/configs.py | 54 ++++++++- tests/lint/multiqc_config.py | 45 ++++++-- tests/test_lint.py | 4 +- 7 files changed, 255 insertions(+), 172 deletions(-) diff --git a/nf_core/lint/configs.py b/nf_core/lint/configs.py index ef8be27e87..2741529198 100644 --- a/nf_core/lint/configs.py +++ b/nf_core/lint/configs.py @@ -3,6 +3,8 @@ from pathlib import Path from typing import Dict, List +from nf_core.lint_utils import ignore_file + log = logging.getLogger(__name__) @@ -14,44 +16,43 @@ def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: """Lint a file and add the result to the passed or failed list.""" - fn = Path(self.wf_path, file_path) passed: List[str] = [] failed: List[str] = [] ignored: List[str] = [] + ignore_configs: List[str] = [] + + fn = Path(self.wf_path, file_path) - ignore_configs = self.lint_config.get(lint_name, []) - - # Return a failed status if we can't find the file - if not fn.is_file(): - if ignore_configs: - return {"ignored": [f"`{file_path}` not found, but it is ignored."]} - else: - return {"failed": [f"`${file_path}` not found"]} - - try: - with open(fn) as fh: - config = fh.read() - except Exception as e: - return {"failed": [f"Could not parse file: {fn}, {e}"]} - - # find sections with a withName: prefix - sections = re.findall(r"['\"](.*)['\"]", config) - - # find all .nf files in the workflow directory - nf_files = list(Path(self.wf_path).rglob("*.nf")) - log.debug(f"found nf_files: {nf_files}") - - # check if withName sections are present in config, but not in workflow files - for section in sections: - if section not in ignore_configs or section.lower() not in ignore_configs: - if not any(section in nf_file.read_text() for nf_file in nf_files): - failed.append( - f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the following workflow files: `{nf_files}`." - ) + passed, failed, ignored, ignore_configs = ignore_file(lint_name, file_path, Path(self.wf_path)) + + error_message = f"`{file_path}` not found" + # check for partial match in failed or ignored + if not any(f.startswith(error_message) for f in (failed + ignored)): + try: + with open(fn) as fh: + config = fh.read() + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]} + + # find sections with a withName: prefix + sections = re.findall(r"withName:\s*['\"]?(\w+)['\"]?", config) + log.debug(f"found sections: {sections}") + + # find all .nf files in the workflow directory + nf_files = list(Path(self.wf_path).rglob("*.nf")) + log.debug(f"found nf_files: {[str(f) for f in nf_files]}") + + # check if withName sections are present in config, but not in workflow files + for section in sections: + if section not in ignore_configs or section.lower() not in ignore_configs: + if not any(section in nf_file.read_text() for nf_file in nf_files): + failed.append( + f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the Nextflow scripts." + ) + else: + passed.append(f"`{section}` found in `{file_path}` and Nextflow scripts.") else: - passed.append(f"both `{file_path}` and `{[str(f) for f in nf_files]} contain `{section}`.") - else: - ignored.append(f"``{section}` is ignored") + ignored.append(f"``{section}` is ignored.") return {"passed": passed, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index 9b9c80c44e..cd281dd5c9 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -1,8 +1,10 @@ -import os +from pathlib import Path from typing import Dict, List import yaml +from nf_core.lint_utils import ignore_file + def multiqc_config(self) -> Dict[str, List[str]]: """Make sure basic multiQC plugins are installed and plots are exported @@ -21,100 +23,110 @@ def multiqc_config(self) -> Dict[str, List[str]]: order: -1001 export_plots: true - """ - - passed: List[str] = [] - failed: List[str] = [] - - # Remove field that should be ignored according to the linting config - ignore_configs = self.lint_config.get("multiqc_config", []) + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: - fn = os.path.join(self.wf_path, "assets", "multiqc_config.yml") + .. code-block:: yaml - # Return a failed status if we can't find the file - if not os.path.isfile(fn): - return {"ignored": ["'assets/multiqc_config.yml' not found"]} + lint: + multiqc_config: False - try: - with open(fn) as fh: - mqc_yml = yaml.safe_load(fh) - except Exception as e: - return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} - - # check if requried sections are present - required_sections = ["report_section_order", "export_plots", "report_comment"] - for section in required_sections: - if section not in mqc_yml and section not in ignore_configs: - failed.append(f"'assets/multiqc_config.yml' does not contain `{section}`") - return {"passed": passed, "failed": failed} - else: - passed.append(f"'assets/multiqc_config.yml' contains `{section}`") - - try: - orders = {} - summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" - min_plugins = ["software_versions", summary_plugin_name] - for plugin in min_plugins: - if plugin not in mqc_yml["report_section_order"]: - raise AssertionError(f"Section {plugin} missing in report_section_order") - if "order" not in mqc_yml["report_section_order"][plugin]: - raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0") - plugin_order = mqc_yml["report_section_order"][plugin]["order"] - if plugin_order >= 0: - raise AssertionError(f"Section {plugin} 'order' must be < 0") - - for plugin in mqc_yml["report_section_order"]: - if "order" in mqc_yml["report_section_order"][plugin]: - orders[plugin] = mqc_yml["report_section_order"][plugin]["order"] - - if orders[summary_plugin_name] != min(orders.values()): - raise AssertionError(f"Section {summary_plugin_name} should have the lowest order") - orders.pop(summary_plugin_name) - if orders["software_versions"] != min(orders.values()): - raise AssertionError("Section software_versions should have the second lowest order") - except (AssertionError, KeyError, TypeError) as e: - failed.append(f"'assets/multiqc_config.yml' does not meet requirements: {e}") - else: - passed.append("'assets/multiqc_config.yml' follows the ordering scheme of the minimally required plugins.") - - if "report_comment" not in ignore_configs: - # Check that the minimum plugins exist and are coming first in the summary - version = self.nf_config.get("manifest.version", "").strip(" '\"") - if "dev" in version: - version = "dev" - report_comments = ( - f'This report has been generated by the nf-core/{self.pipeline_name}' - f" analysis pipeline. For information about how to interpret these results, please see the " - f'documentation.' - ) + """ + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + + fn = Path(self.wf_path, "assets", "multiqc_config.yml") + file_path = fn.relative_to(self.wf_path) + passed, failed, ignored, ignore_configs = ignore_file("multiqc_config", file_path, self.wf_path) + print(f"passed: {passed}") + print(f"failed: {failed}") + print(f"ignored: {ignored}") + # skip other tests if the file is not found + error_message = f"`{file_path}` not found" + # check for partial match in failed or ignored + if not any(f.startswith(error_message) for f in (failed + ignored)): + try: + with open(fn) as fh: + mqc_yml = yaml.safe_load(fh) + except Exception as e: + return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} + + # check if required sections are present + required_sections = ["report_section_order", "export_plots", "report_comment"] + for section in required_sections: + if section not in mqc_yml and section not in ignore_configs: + failed.append(f"`assets/multiqc_config.yml` does not contain `{section}`") + return {"passed": passed, "failed": failed} + else: + passed.append(f"`assets/multiqc_config.yml` contains `{section}`") + + try: + orders = {} + summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" + min_plugins = ["software_versions", summary_plugin_name] + for plugin in min_plugins: + if plugin not in mqc_yml["report_section_order"]: + raise AssertionError(f"Section {plugin} missing in report_section_order") + if "order" not in mqc_yml["report_section_order"][plugin]: + raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0") + plugin_order = mqc_yml["report_section_order"][plugin]["order"] + if plugin_order >= 0: + raise AssertionError(f"Section {plugin} 'order' must be < 0") + + for plugin in mqc_yml["report_section_order"]: + if "order" in mqc_yml["report_section_order"][plugin]: + orders[plugin] = mqc_yml["report_section_order"][plugin]["order"] + + if orders[summary_plugin_name] != min(orders.values()): + raise AssertionError(f"Section {summary_plugin_name} should have the lowest order") + orders.pop(summary_plugin_name) + if orders["software_versions"] != min(orders.values()): + raise AssertionError("Section software_versions should have the second lowest order") + except (AssertionError, KeyError, TypeError) as e: + failed.append(f"`assets/multiqc_config.yml` does not meet requirements: {e}") else: - report_comments = ( - f'This report has been generated by the nf-core/{self.pipeline_name}' - f" analysis pipeline. For information about how to interpret these results, please see the " - f'documentation.' - ) - - if mqc_yml["report_comment"].strip() != report_comments: - # find where the report_comment is wrong and give it as a hint - hint = report_comments - failed.append( - f"'assets/multiqc_config.yml' does not contain a matching 'report_comment'. \n" - f"The expected comment is: \n" - f"```{hint}``` \n" - f"The current comment is: \n" - f"```{ mqc_yml['report_comment'].strip()}```" - ) + passed.append("`assets/multiqc_config.yml` follows the ordering scheme of the minimally required plugins.") + + if "report_comment" not in ignore_configs: + # Check that the minimum plugins exist and are coming first in the summary + version = self.nf_config.get("manifest.version", "").strip(" '\"") + if "dev" in version: + version = "dev" + report_comments = ( + f'This report has been generated by the nf-core/{self.pipeline_name}' + f" analysis pipeline. For information about how to interpret these results, please see the " + f'documentation.' + ) + + else: + report_comments = ( + f'This report has been generated by the nf-core/{self.pipeline_name}' + f" analysis pipeline. For information about how to interpret these results, please see the " + f'documentation.' + ) + + if mqc_yml["report_comment"].strip() != report_comments: + # find where the report_comment is wrong and give it as a hint + hint = report_comments + failed.append( + f"`assets/multiqc_config.yml` does not contain a matching 'report_comment'. \n" + f"The expected comment is: \n" + f"```{hint}``` \n" + f"The current comment is: \n" + f"```{ mqc_yml['report_comment'].strip()}```" + ) + else: + passed.append("`assets/multiqc_config.yml` contains a matching 'report_comment'.") + + # Check that export_plots is activated + try: + if not mqc_yml["export_plots"]: + raise AssertionError() + except (AssertionError, KeyError, TypeError): + failed.append("`assets/multiqc_config.yml` does not contain 'export_plots: true'.") else: - passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.") - - # Check that export_plots is activated - try: - if not mqc_yml["export_plots"]: - raise AssertionError() - except (AssertionError, KeyError, TypeError): - failed.append("'assets/multiqc_config.yml' does not contain 'export_plots: true'.") - else: - passed.append("'assets/multiqc_config.yml' contains 'export_plots: true'.") - - return {"passed": passed, "failed": failed} + passed.append("`assets/multiqc_config.yml` contains 'export_plots: true'.") + + return {"passed": passed, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 27fb36568f..0ca84c46aa 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -2,7 +2,7 @@ import logging import subprocess from pathlib import Path -from typing import Tuple +from typing import List import rich from rich.console import Console @@ -104,32 +104,30 @@ def dump_json_with_prettier(file_name, file_content): run_prettier_on_file(file_name) -def parse_config_file(self, lint_name: str, file_path: Path) -> Tuple[dict, dict]: - """Parse different kind of config files and return a dict.""" - - # Remove field that should be ignored according to the linting config - ignore_configs = self.lint_config.get(lint_name, []) - - fn = Path(self.wf_path, file_path) +def ignore_file(lint_name: str, file_path: Path, dir_path: Path) -> List[List[str]]: + """Ignore a file and add the result to the ignored list. Return the passed, failed, ignored and ignore_configs lists.""" + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + _, lint_conf = nf_core.utils.load_tools_config(dir_path) + lint_conf = lint_conf.get("lint", {}) + print(f"lint_conf: {lint_conf}") + ignore_entry: List[str] | bool = lint_conf.get(lint_name, []) + print(f"ignore_entry: {ignore_entry}") + full_path = dir_path / file_path # Return a failed status if we can't find the file - if not fn.is_file(): - return {"ignored": [f"`${file_path}` not found"]}, ignore_configs - - try: - if fn.suffix == ".json": - import json - - with open(fn) as fh: - config = json.load(fh) - return config, ignore_configs - elif fn.suffix == ".yml" or fn.suffix == ".yaml": - import yaml - - with open(fn) as fh: - config = yaml.safe_load(fh) - return config, ignore_configs + if not full_path.is_file(): + if isinstance(ignore_entry, bool) and not ignore_entry: + ignored.append(f"`{file_path}` not found, but it is ignored.") + ignore_entry = [] else: - return {"failed": [f"Could not parse file: {fn}, unknown file type"]}, ignore_configs - except Exception as e: - return {"failed": [f"Could not parse file: {fn}, {e}"]}, ignore_configs + failed.append(f"`{file_path}` not found.") + else: + passed.append(f"`{file_path}` found and not ignored.") + + # we handled the only case where ignore_entry should be a bool, convert it to a list, to make downstream code easier + if isinstance(ignore_entry, bool): + ignore_entry = [] + + return [passed, failed, ignored, ignore_entry] diff --git a/nf_core/utils.py b/nf_core/utils.py index 83136c114e..6af2f25165 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -213,7 +213,11 @@ def is_pipeline_directory(wf_path): for fn in ["main.nf", "nextflow.config"]: path = os.path.join(wf_path, fn) if not os.path.isfile(path): - raise UserWarning(f"'{wf_path}' is not a pipeline - '{fn}' is missing") + if wf_path == ".": + warning = f"Current directory is not a pipeline - '{fn}' is missing." + else: + warning = f"'{wf_path}' is not a pipeline - '{fn}' is missing." + raise UserWarning(warning) def fetch_wf_config(wf_path: str, cache_config: bool = True) -> dict: @@ -1013,7 +1017,7 @@ def get_repo_releases_branches(pipeline, wfs): DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"] -def load_tools_config(directory: Union[str, Path] = "."): +def load_tools_config(directory: Union[str, Path] = ".") -> Tuple[Path, dict]: """ Parse the nf-core.yml configuration file @@ -1041,7 +1045,6 @@ def load_tools_config(directory: Union[str, Path] = "."): with open(config_fn) as fh: tools_config = yaml.safe_load(fh) - # If the file is empty tools_config = tools_config or {} diff --git a/tests/lint/configs.py b/tests/lint/configs.py index a4bbe0147c..d5dcd27fad 100644 --- a/tests/lint/configs.py +++ b/tests/lint/configs.py @@ -1,5 +1,7 @@ from pathlib import Path +import yaml + import nf_core.create import nf_core.lint @@ -12,7 +14,9 @@ def test_withname_in_modules_config(self): lint_obj._load() result = lint_obj.modules_config() assert len(result["failed"]) == 0 - assert any(["contain `FASTQC`" in passed for passed in result["passed"]]) + assert any( + ["`FASTQC` found in `conf/modules.config` and Nextflow scripts." in passed for passed in result["passed"]] + ) def test_superfluous_withname_in_modules_config_fails(self): @@ -21,12 +25,32 @@ def test_superfluous_withname_in_modules_config_fails(self): # Add withName to modules.config modules_config = Path(new_pipeline) / "conf" / "modules.config" with open(modules_config, "a") as f: - f.write("withName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") - lint_obj = nf_core.lint.PipelineLint(new_pipeline) + f.write("\nwithName: 'BPIPE' {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline, hide_progress=False) lint_obj._load() result = lint_obj.modules_config() assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("`conf/modules.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") + assert result["failed"][0].startswith("`conf/modules.config` contains `withName:BPIPE`") + + +def test_ignore_modules_config(self): + """Tests ignoring the modules.config passes linting.""" + new_pipeline = self._make_pipeline_copy() + # ignore modules.config in linting + with open(Path(new_pipeline) / ".nf-core.yml") as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"modules_config": False} + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(content, f) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["ignored"]) == 1 + assert result["ignored"][0].startswith("`conf/modules.config` not found, but it is ignored.") + # cleanup + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(old_content, f) def test_superfluous_withname_in_base_config_fails(self): @@ -35,9 +59,29 @@ def test_superfluous_withname_in_base_config_fails(self): # Add withName to base.config base_config = Path(new_pipeline) / "conf" / "base.config" with open(base_config, "a") as f: - f.write("withName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") + f.write("\nwithName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() result = lint_obj.base_config() assert len(result["failed"]) == 1 assert result["failed"][0].startswith("`conf/base.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") + + +def test_ignore_base_config(self): + """Tests ignoring the base.config passes linting.""" + new_pipeline = self._make_pipeline_copy() + # ignore base.config in linting + with open(Path(new_pipeline) / ".nf-core.yml") as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"base_config": False} + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(content, f) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.base_config() + assert len(result["ignored"]) == 1 + assert result["ignored"][0].startswith("`conf/base.config` not found, but it is ignored.") + # cleanup + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(old_content, f) diff --git a/tests/lint/multiqc_config.py b/tests/lint/multiqc_config.py index 721560ce81..463d5e7654 100644 --- a/tests/lint/multiqc_config.py +++ b/tests/lint/multiqc_config.py @@ -5,14 +5,37 @@ import nf_core.lint -def test_multiqc_config_exists_ignore(self): +def test_multiqc_config_exists(self): """Test that linting fails if the multiqc_config.yml file is missing""" # Delete the file new_pipeline = self._make_pipeline_copy() Path(Path(new_pipeline, "assets", "multiqc_config.yml")).unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() result = lint_obj.multiqc_config() - assert result["ignored"] == ["'assets/multiqc_config.yml' not found"] + assert result["failed"] == ["`assets/multiqc_config.yml` not found."] + + +def test_multiqc_config_ignore(self): + """Test that linting succeeds if the multiqc_config.yml file is missing but ignored""" + # Delete the file + new_pipeline = self._make_pipeline_copy() + Path(Path(new_pipeline, "assets", "multiqc_config.yml")).unlink() + with open(Path(new_pipeline, ".nf-core.yml")) as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"multiqc_config": False} + with open(Path(new_pipeline, ".nf-core.yml"), "w") as f: + yaml.dump(content, f) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.multiqc_config() + assert result["ignored"] == ["`assets/multiqc_config.yml` not found, but it is ignored."] + + # cleanup + with open(Path(new_pipeline, ".nf-core.yml"), "w") as f: + yaml.dump(old_content, f) def test_multiqc_config_missing_report_section_order(self): @@ -20,7 +43,7 @@ def test_multiqc_config_missing_report_section_order(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml.pop("report_section_order") with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -30,7 +53,7 @@ def test_multiqc_config_missing_report_section_order(self): # Reset the file with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) - assert result["failed"] == ["'assets/multiqc_config.yml' does not contain `report_section_order`"] + assert result["failed"] == ["`assets/multiqc_config.yml` does not contain `report_section_order`"] def test_multiqc_incorrect_export_plots(self): @@ -38,7 +61,7 @@ def test_multiqc_incorrect_export_plots(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml["export_plots"] = False with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -48,7 +71,7 @@ def test_multiqc_incorrect_export_plots(self): # Reset the file with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) - assert result["failed"] == ["'assets/multiqc_config.yml' does not contain 'export_plots: true'."] + assert result["failed"] == ["`assets/multiqc_config.yml` does not contain 'export_plots: true'."] def test_multiqc_config_report_comment_fail(self): @@ -56,7 +79,7 @@ def test_multiqc_config_report_comment_fail(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml["report_comment"] = "This is a test" with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -67,7 +90,7 @@ def test_multiqc_config_report_comment_fail(self): with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + assert result["failed"][0].startswith("`assets/multiqc_config.yml` does not contain a matching 'report_comment'.") def test_multiqc_config_report_comment_release_fail(self): @@ -75,7 +98,7 @@ def test_multiqc_config_report_comment_release_fail(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -87,7 +110,7 @@ def test_multiqc_config_report_comment_release_fail(self): with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + assert result["failed"][0].startswith("`assets/multiqc_config.yml` does not contain a matching 'report_comment'.") def test_multiqc_config_report_comment_release_succeed(self): @@ -103,4 +126,4 @@ def test_multiqc_config_report_comment_release_succeed(self): # lint again lint_obj._load() result = lint_obj.multiqc_config() - assert "'assets/multiqc_config.yml' contains a matching 'report_comment'." in result["passed"] + assert "`assets/multiqc_config.yml` contains a matching 'report_comment'." in result["passed"] diff --git a/tests/test_lint.py b/tests/test_lint.py index 290b9b73c7..51c52fd54e 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -199,6 +199,7 @@ def test_sphinx_md_files(self): test_actions_schema_validation_missing_on, ) from .lint.configs import ( # type: ignore[misc] + test_ignore_modules_config, test_superfluous_withname_in_base_config_fails, test_superfluous_withname_in_modules_config_fails, test_withname_in_modules_config, @@ -218,7 +219,8 @@ def test_sphinx_md_files(self): from .lint.merge_markers import test_merge_markers_found # type: ignore[misc] from .lint.modules_json import test_modules_json_pass # type: ignore[misc] from .lint.multiqc_config import ( # type: ignore[misc] - test_multiqc_config_exists_ignore, + test_multiqc_config_exists, + test_multiqc_config_ignore, test_multiqc_config_missing_report_section_order, test_multiqc_config_report_comment_fail, test_multiqc_config_report_comment_release_fail, From cac553c5f1d7d5be3a98244b1d6d6e1ccfb655d9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 18:16:45 +0000 Subject: [PATCH 074/142] Update gitpod/workspace-base Docker digest to 450d764 --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 40aa998a50..e476a7f140 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:168d78bc249332f9437b11a1844819308ac7ec1bc6bab61bf5adf8cbe53bf8e6 +FROM gitpod/workspace-base@sha256:450d764c8575345b32e3e22d197b5150152eff9f9a14482003f70f05d03f726a USER root From 0f9da5d421947f9a7fb72dccced41cd1257940a0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 3 Apr 2024 09:39:38 +0200 Subject: [PATCH 075/142] fix tests --- nf_core/lint/multiqc_config.py | 4 +--- nf_core/lint_utils.py | 2 -- tests/lint/configs.py | 2 ++ tests/test_lint.py | 1 + 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index cd281dd5c9..8b4fa2120f 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -40,9 +40,7 @@ def multiqc_config(self) -> Dict[str, List[str]]: fn = Path(self.wf_path, "assets", "multiqc_config.yml") file_path = fn.relative_to(self.wf_path) passed, failed, ignored, ignore_configs = ignore_file("multiqc_config", file_path, self.wf_path) - print(f"passed: {passed}") - print(f"failed: {failed}") - print(f"ignored: {ignored}") + # skip other tests if the file is not found error_message = f"`{file_path}` not found" # check for partial match in failed or ignored diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 0ca84c46aa..167600bfc0 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -112,9 +112,7 @@ def ignore_file(lint_name: str, file_path: Path, dir_path: Path) -> List[List[st ignored: List[str] = [] _, lint_conf = nf_core.utils.load_tools_config(dir_path) lint_conf = lint_conf.get("lint", {}) - print(f"lint_conf: {lint_conf}") ignore_entry: List[str] | bool = lint_conf.get(lint_name, []) - print(f"ignore_entry: {ignore_entry}") full_path = dir_path / file_path # Return a failed status if we can't find the file if not full_path.is_file(): diff --git a/tests/lint/configs.py b/tests/lint/configs.py index d5dcd27fad..b50a1393aa 100644 --- a/tests/lint/configs.py +++ b/tests/lint/configs.py @@ -43,6 +43,7 @@ def test_ignore_modules_config(self): content["lint"] = {"modules_config": False} with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: yaml.dump(content, f) + Path(new_pipeline, "conf", "modules.config").unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() result = lint_obj.modules_config() @@ -77,6 +78,7 @@ def test_ignore_base_config(self): content["lint"] = {"base_config": False} with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: yaml.dump(content, f) + Path(new_pipeline, "conf", "base.config").unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() result = lint_obj.base_config() diff --git a/tests/test_lint.py b/tests/test_lint.py index 1af5c40461..b72a6bfdfa 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -199,6 +199,7 @@ def test_sphinx_md_files(self): test_actions_schema_validation_missing_on, ) from .lint.configs import ( # type: ignore[misc] + test_ignore_base_config, test_ignore_modules_config, test_superfluous_withname_in_base_config_fails, test_superfluous_withname_in_modules_config_fails, From c29513766b21d4444b904c619f954867f8614fb6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 3 Apr 2024 18:56:52 +0200 Subject: [PATCH 076/142] fix tests --- nf_core/modules/lint/module_tests.py | 28 +++++++++---------- .../subworkflows/lint/subworkflow_tests.py | 28 +++++++++---------- tests/test_modules.py | 2 +- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 52cbaf04e5..b2b6c2221f 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -107,22 +107,22 @@ def module_tests(_, module: NFCoreComponent): snap_file, ) ) - if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): - module.passed.append( - ( - "test_snap_versions", - "versions found in snapshot file", - snap_file, - ) + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): + module.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, ) - else: - module.failed.append( - ( - "test_snap_versions", - "versions not found in snapshot file", - snap_file, - ) + ) + else: + module.failed.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, ) + ) except json.decoder.JSONDecodeError as e: module.failed.append( ( diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 30d4296aac..02a429b661 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -114,22 +114,22 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): snap_file, ) ) - if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): - subworkflow.passed.append( - ( - "test_snap_versions", - "versions found in snapshot file", - snap_file, - ) + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): + subworkflow.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, ) - else: - subworkflow.warned.append( - ( - "test_snap_versions", - "versions not found in snapshot file", - snap_file, - ) + ) + else: + subworkflow.warned.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, ) + ) except json.decoder.JSONDecodeError as e: subworkflow.failed.append( ( diff --git a/tests/test_modules.py b/tests/test_modules.py index 26fdea41ac..354bd2bb28 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -55,7 +55,7 @@ def create_modules_repo_dummy(tmp_dir): test_snap_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") test_snap_path.touch() with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n') + fh.write('{\n "my test": {},\n') fh.write(' "versions": {}\n}') # remove "TODO" statements from main.nf From c554caed7a396bf9deb61526e66591b69771d8e9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 07:56:01 +0200 Subject: [PATCH 077/142] fix subworkflow tests --- tests/subworkflows/lint.py | 18 +++++++++--------- tests/test_subworkflows.py | 5 +++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index a315ea3912..224278a5c8 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -185,20 +185,20 @@ def test_subworkflows_lint_capitalization_fail(self): def test_subworkflows_absent_version(self): """Test linting a nf-test module if the versions is absent in the snapshot file `""" - self.subworkflow_install.install("bam_sort_stats_samtools") - with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") - ) as fh: + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + with open(snap_file) as fh: content = fh.read() new_content = content.replace("versions", "foo") - with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap"), "w" - ) as fh: + with open(snap_file, "w") as fh: fh.write(new_content) - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, subworkflow="bam_sort_stats_samtools") + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 0 assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" assert any([x.lint_test == "test_snap_versions" for x in subworkflow_lint.warned]) + + # cleanup + with open(snap_file, "w") as fh: + fh.write(content) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 775a08547f..329156227d 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -23,7 +23,6 @@ def create_modules_repo_dummy(tmp_dir): root_dir = Path(tmp_dir, "modules") Path(root_dir, "modules").mkdir(parents=True, exist_ok=True) - Path(root_dir, "subworkflows").mkdir(parents=True, exist_ok=True) Path(root_dir, "subworkflows", "nf-core").mkdir(parents=True, exist_ok=True) Path(root_dir, "tests", "config").mkdir(parents=True, exist_ok=True) with open(Path(root_dir, ".nf-core.yml"), "w") as fh: @@ -34,9 +33,11 @@ def create_modules_repo_dummy(tmp_dir): # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + test_snap_path.parent.mkdir(parents=True, exist_ok=True) test_snap_path.touch() with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n}') + fh.write('{\n "my test": {},\n') + fh.write(' "versions": {}\n}') return root_dir From 3ae9d0e32b78eb1805ef5fd925ef54814dbc2006 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Thu, 4 Apr 2024 13:11:41 +0200 Subject: [PATCH 078/142] Shorten lines in pipeline template --- .../pipeline-template/workflows/pipeline.nf | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 68adbaa328..8cdff1baa3 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -40,22 +40,44 @@ workflow {{ short_name|upper }} { // Collate and save software versions // softwareVersionsToYAML(ch_versions) - .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) - .set { ch_collated_versions } + .collectFile( + storeDir: "${params.outdir}/pipeline_info", + name: 'nf_core_pipeline_software_mqc_versions.yml', + sort: true, + newLine: true + ).set { ch_collated_versions } // // MODULE: MultiQC // - ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) - ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() - ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") - ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) - ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) - ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) - ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) - ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false)) + ch_multiqc_config = Channel.fromPath( + "$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = params.multiqc_config ? + Channel.fromPath(params.multiqc_config, checkIfExists: true) : + Channel.empty() + ch_multiqc_logo = params.multiqc_logo ? + Channel.fromPath(params.multiqc_logo, checkIfExists: true) : + Channel.empty() + + summary_params = paramsSummaryMap( + workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + + ch_multiqc_custom_methods_description = params.multiqc_methods_description ? + file(params.multiqc_methods_description, checkIfExists: true) : + file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_methods_description = Channel.value( + methodsDescriptionText(ch_multiqc_custom_methods_description)) + + ch_multiqc_files = ch_multiqc_files.mix( + ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_files = ch_multiqc_files.mix( + ch_methods_description.collectFile( + name: 'methods_description_mqc.yaml', + sort: false + ) + ) MULTIQC ( ch_multiqc_files.collect(), From 4dee9d90d1aa58b9a0c765155bfa7ec66a70936c Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 4 Apr 2024 12:06:16 +0000 Subject: [PATCH 079/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ad053f6df..3893b1860e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Add a cleanup action to `download_pipeline.yml` to fix failures caused by inadequate storage space on the runner ([#2849](https://github.com/nf-core/tools/pull/2849)) - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root +- Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) ### Linting From 9df6e378a8e024c9d334abf36bf2f26d0df2357f Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 16:19:38 +0200 Subject: [PATCH 080/142] fix snapshot test to handle updated fastp version --- tests/components/generate_snapshot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index 50024a8ebb..e42aeaa66e 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -1,6 +1,7 @@ """Test generate a snapshot""" import json +import shutil from pathlib import Path from unittest.mock import MagicMock @@ -120,9 +121,12 @@ def test_test_not_found(self): remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) + test_file = Path("modules", "nf-core-test", "fastp", "tests", "main.nf.test") + shutil.move(test_file, test_file.parent / "main.nf.test.bak") with pytest.raises(UserWarning) as e: snap_generator.run() assert "Test file 'main.nf.test' not found" in str(e.value) + shutil.move(test_file.parent / "main.nf.test.bak", test_file) def test_unstable_snapshot(self): From 70fac756f53bd74114caf24c6dd11817ad44658c Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 16:28:52 +0200 Subject: [PATCH 081/142] add linting tests for empty files in snapshots --- tests/components/generate_snapshot.py | 5 +- tests/modules/lint.py | 45 ++++++++++++ tests/subworkflows/lint.py | 100 ++++++++++++++++++++++++++ tests/test_modules.py | 21 +++++- tests/test_subworkflows.py | 20 +++++- 5 files changed, 182 insertions(+), 9 deletions(-) diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index e42aeaa66e..3176569ec8 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -1,7 +1,6 @@ """Test generate a snapshot""" import json -import shutil from pathlib import Path from unittest.mock import MagicMock @@ -122,11 +121,11 @@ def test_test_not_found(self): branch=GITLAB_NFTEST_BRANCH, ) test_file = Path("modules", "nf-core-test", "fastp", "tests", "main.nf.test") - shutil.move(test_file, test_file.parent / "main.nf.test.bak") + test_file.rename(test_file.parent / "main.nf.test.bak") with pytest.raises(UserWarning) as e: snap_generator.run() assert "Test file 'main.nf.test' not found" in str(e.value) - shutil.move(test_file.parent / "main.nf.test.bak", test_file) + Path(test_file.parent / "main.nf.test.bak").rename(test_file) def test_unstable_snapshot(self): diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 5e3bcfbd30..265caed379 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -1,3 +1,4 @@ +import json from pathlib import Path import pytest @@ -663,3 +664,47 @@ def test_modules_absent_version(self): assert len(module_lint.passed) >= 0 assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "test_snap_versions" + + +def test_modules_empty_file_in_snapshot(self): + """Test linting a nf-test module with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" + snap_file = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_snap_md5sum" + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_modules_empty_file_in_stub_snapshot(self): + """Test linting a nf-test module with an empty file sha sum in the stub test snapshot, which should make it not fail""" + snap_file = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert any(x.lint_test == "test_snap_md5sum" for x in module_lint.passed) + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 224278a5c8..28d29d2f9d 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -1,3 +1,5 @@ +import json +import shutil from pathlib import Path import pytest @@ -202,3 +204,101 @@ def test_subworkflows_absent_version(self): # cleanup with open(snap_file, "w") as fh: fh.write(content) + + +def test_subworkflows_missing_test_dir(self): + """Test linting a nf-test subworkflow if the tests directory is missing""" + test_dir = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests") + test_dir_copy = shutil.copytree(test_dir, test_dir.parent / "tests_copy") + shutil.rmtree(test_dir) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" + assert any([x.lint_test == "test_dir_versions" for x in subworkflow_lint.warned]) + + # cleanup + shutil.copytree(test_dir_copy, test_dir) + + +def test_subworkflows_missing_main_nf(self): + """Test linting a nf-test subworkflow if the main.nf file is missing""" + main_nf = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf") + main_nf_copy = shutil.copy(main_nf, main_nf.parent / "main_nf_copy") + main_nf.unlink() + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "main_nf_exists" + + # cleanup + shutil.copy(main_nf_copy, main_nf) + + +def test_subworkflows_empty_file_in_snapshot(self): + """Test linting a nf-test subworkflow with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" + snap_file = Path( + self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap" + ) + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "test_snap_md5sum" + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_empty_file_in_stub_snapshot(self): + """Test linting a nf-test subworkflow with an empty file sha sum in the stub test snapshot, which should make it not fail""" + snap_file = Path( + self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap" + ) + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert any(x.lint_test == "test_snap_md5sum" for x in subworkflow_lint.passed) + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_missing_tags_yml(self): + """Test linting a subworkflow with a missing tags.yml file""" + tags_path = Path(self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + tags_path.rename(tags_path.parent / "tags.yml.bak") + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) >= 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "test_tags_yml_exists" + + # cleanup + tags_path.rename(tags_path.parent / "tags.yml") diff --git a/tests/test_modules.py b/tests/test_modules.py index 354bd2bb28..d3d99abadd 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,5 +1,6 @@ """Tests covering the modules commands""" +import json import os import shutil import unittest @@ -53,10 +54,22 @@ def create_modules_repo_dummy(tmp_dir): yaml.dump(meta_yml, fh) # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") - test_snap_path.touch() + with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {},\n') - fh.write(' "versions": {}\n}') + json.dump( + { + "my test": { + "content": [ + { + "0": [], + "versions": {}, + } + ] + } + }, + fh, + indent=4, + ) # remove "TODO" statements from main.nf main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") @@ -181,6 +194,8 @@ def test_modulesrepo_class(self): ) from .modules.lint import ( # type: ignore[misc] test_modules_absent_version, + test_modules_empty_file_in_snapshot, + test_modules_empty_file_in_stub_snapshot, test_modules_environment_yml_file_doesnt_exists, test_modules_environment_yml_file_name_mismatch, test_modules_environment_yml_file_not_array, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 329156227d..6c00df9816 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -1,5 +1,6 @@ """Tests covering the subworkflows commands""" +import json import os import shutil import unittest @@ -34,10 +35,21 @@ def create_modules_repo_dummy(tmp_dir): # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") test_snap_path.parent.mkdir(parents=True, exist_ok=True) - test_snap_path.touch() with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {},\n') - fh.write(' "versions": {}\n}') + json.dump( + { + "my test": { + "content": [ + { + "0": [], + "versions": {}, + } + ] + } + }, + fh, + indent=4, + ) return root_dir @@ -121,6 +133,8 @@ def tearDown(self): ) from .subworkflows.lint import ( # type: ignore[misc] test_subworkflows_absent_version, + test_subworkflows_empty_file_in_snapshot, + test_subworkflows_empty_file_in_stub_snapshot, test_subworkflows_lint, test_subworkflows_lint_capitalization_fail, test_subworkflows_lint_empty, From 953a980e5c2ad38ca702304c0b2f4e05d2687450 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 17:56:07 +0200 Subject: [PATCH 082/142] add more tags.yml tests for subworkflows --- tests/modules/lint.py | 24 +++++++-------- tests/subworkflows/lint.py | 60 +++++++++++++++++++++++++++++++------- tests/test_subworkflows.py | 3 ++ 3 files changed, 64 insertions(+), 23 deletions(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 265caed379..de7ec203ef 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -564,30 +564,29 @@ def test_modules_missing_required_tag(self): def test_modules_missing_tags_yml(self): """Test linting a module with a missing tags.yml file""" - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml").rename( - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak") - ) + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + tags_path.rename(tags_path.parent / "tags.yml.bak") module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak").rename( - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") - ) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "test_tags_yml_exists" + # cleanup + Path(tags_path.parent / "tags.yml").rename(tags_path.parent / "tags.yml") def test_modules_incorrect_tags_yml_key(self): """Test linting a module with an incorrect key in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + with open(tags_path) as fh: content = fh.read() new_content = content.replace("bpipe/test:", "bpipe_test:") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(new_content) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=True, module="bpipe/test") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(content) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 @@ -597,14 +596,15 @@ def test_modules_incorrect_tags_yml_key(self): def test_modules_incorrect_tags_yml_values(self): """Test linting a module with an incorrect path in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + with open(tags_path) as fh: content = fh.read() new_content = content.replace("modules/nf-core/bpipe/test/**", "foo") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(new_content) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(content) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 28d29d2f9d..73d2452b37 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -242,9 +242,7 @@ def test_subworkflows_missing_main_nf(self): def test_subworkflows_empty_file_in_snapshot(self): """Test linting a nf-test subworkflow with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" - snap_file = Path( - self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap" - ) + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") snap = json.load(snap_file.open()) content = snap_file.read_text() snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" @@ -252,7 +250,7 @@ def test_subworkflows_empty_file_in_snapshot(self): with open(snap_file, "w") as fh: json.dump(snap, fh) - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 @@ -266,9 +264,7 @@ def test_subworkflows_empty_file_in_snapshot(self): def test_subworkflows_empty_file_in_stub_snapshot(self): """Test linting a nf-test subworkflow with an empty file sha sum in the stub test snapshot, which should make it not fail""" - snap_file = Path( - self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap" - ) + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") snap = json.load(snap_file.open()) content = snap_file.read_text() snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} @@ -276,7 +272,7 @@ def test_subworkflows_empty_file_in_stub_snapshot(self): with open(snap_file, "w") as fh: json.dump(snap, fh) - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 @@ -290,9 +286,9 @@ def test_subworkflows_empty_file_in_stub_snapshot(self): def test_subworkflows_missing_tags_yml(self): """Test linting a subworkflow with a missing tags.yml file""" - tags_path = Path(self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") tags_path.rename(tags_path.parent / "tags.yml.bak") - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" @@ -301,4 +297,46 @@ def test_subworkflows_missing_tags_yml(self): assert subworkflow_lint.failed[0].lint_test == "test_tags_yml_exists" # cleanup - tags_path.rename(tags_path.parent / "tags.yml") + Path(tags_path.parent / "tags.yml.bak").rename(tags_path.parent / "tags.yml") + + +def test_subworkflows_incorrect_tags_yml_key(self): + """Test linting a subworkflow with an incorrect key in tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + with open(tags_path) as fh: + content = fh.read() + new_content = content.replace("test_subworkflow:", "subworkflow:") + with open(tags_path, "w") as fh: + fh.write(new_content) + module_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + module_lint.lint(print_results=True, subworkflow="test_subworkflow") + with open(tags_path, "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + # cleanup + with open(tags_path, "w") as fh: + fh.write(content) + + +def test_subworkflows_incorrect_tags_yml_values(self): + """Test linting a subworkflow with an incorrect path in tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + with open(tags_path) as fh: + content = fh.read() + new_content = content.replace("subworkflows/nf-core/test_subworkflow/**", "foo") + with open(tags_path, "w") as fh: + fh.write(new_content) + module_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, subworkflow="test_subworkflow") + with open(tags_path, "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + # cleanup + with open(tags_path, "w") as fh: + fh.write(content) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 6c00df9816..0a9224002a 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -135,6 +135,8 @@ def tearDown(self): test_subworkflows_absent_version, test_subworkflows_empty_file_in_snapshot, test_subworkflows_empty_file_in_stub_snapshot, + test_subworkflows_incorrect_tags_yml_key, + test_subworkflows_incorrect_tags_yml_values, test_subworkflows_lint, test_subworkflows_lint_capitalization_fail, test_subworkflows_lint_empty, @@ -147,6 +149,7 @@ def tearDown(self): test_subworkflows_lint_snapshot_file, test_subworkflows_lint_snapshot_file_missing_fail, test_subworkflows_lint_snapshot_file_not_needed, + test_subworkflows_missing_tags_yml, ) from .subworkflows.list import ( # type: ignore[misc] test_subworkflows_install_and_list_subworkflows, From 53c7cda0b220ac5dd1170530f9b3658b6bea6844 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 18:12:09 +0200 Subject: [PATCH 083/142] fix test --- tests/modules/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index de7ec203ef..595509de4e 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -573,7 +573,7 @@ def test_modules_missing_tags_yml(self): assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "test_tags_yml_exists" # cleanup - Path(tags_path.parent / "tags.yml").rename(tags_path.parent / "tags.yml") + Path(tags_path.parent / "tags.yml.bak").rename(tags_path.parent / "tags.yml") def test_modules_incorrect_tags_yml_key(self): From 986f9d07adb51b7ffaabd64e771ca23f9c2c50bc Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 4 Apr 2024 00:20:08 +0000 Subject: [PATCH 084/142] chore(deps): update codecov/codecov-action digest to 7afa10e --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 93a44aea5e..c6af0ed983 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -184,7 +184,7 @@ jobs: coverage report coverage xml - - uses: codecov/codecov-action@c16abc29c95fcf9174b58eb7e1abf4c866893bc8 # v4 + - uses: codecov/codecov-action@7afa10ed9b269c561c2336fd862446844e0cbf71 # v4 with: files: coverage.xml env: From b9e4d2fb5edea183db8d06ba9daab63e1cdf0958 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 5 Apr 2024 03:35:34 +0000 Subject: [PATCH 085/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3893b1860e..9c5a874df9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ - Get immediate parent path name for schema creation ([#2886](https://github.com/nf-core/tools/pull/2886)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 ([#2903](https://github.com/nf-core/tools/pull/2903)) - Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897)) +- Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 32aac4d96adab4edb147c43613a74c542381d918 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 04:44:50 +0000 Subject: [PATCH 086/142] Update codecov/codecov-action digest to 8450866 --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c6af0ed983..1303409c05 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -184,7 +184,7 @@ jobs: coverage report coverage xml - - uses: codecov/codecov-action@7afa10ed9b269c561c2336fd862446844e0cbf71 # v4 + - uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # v4 with: files: coverage.xml env: From b75416da9a493fa967289f845e0caa6b8c7146f4 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 11 Apr 2024 06:22:30 +0000 Subject: [PATCH 087/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 159a283daa..2da49c4829 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 ([#2903](https://github.com/nf-core/tools/pull/2903)) - Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897)) - Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) +- Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 513b5feb6f118535b29f084ca03b1df418077f0a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 09:51:03 +0000 Subject: [PATCH 088/142] chore(deps): update gitpod/workspace-base docker digest to 674315a --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index e476a7f140..ca69f74f71 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:450d764c8575345b32e3e22d197b5150152eff9f9a14482003f70f05d03f726a +FROM gitpod/workspace-base@sha256:674315a74b11e7b796040587d5e5ad000d755e0f14dd070704757e578be25fd9 USER root From b1c19fda298cadc7aa4b337dc349804364d8e918 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:43:44 +0000 Subject: [PATCH 089/142] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 915bbc345d..d7fed0449d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.5 + rev: v0.3.6 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 123ffcd8593e6807f93ef4892c83dfde0f375452 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:43:41 +0000 Subject: [PATCH 090/142] chore(deps): update python:3.12-slim docker digest to 541d45d --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f49001089..c801563667 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:5dc6f84b5e97bfb0c90abfb7c55f3cacc2cb6687c8f920b64a833a2219875997 +FROM python:3.12-slim@sha256:541d45d3d675fb8197f534525a671e2f8d66c882b89491f9dda271f4f94dcd06 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From 0932fd2624fb3aff33ecc07d00845865a4e24005 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 11 Apr 2024 17:44:36 +0000 Subject: [PATCH 091/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2da49c4829..7e758c6e60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ - Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897)) - Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) - Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 104d5b4c6909dc3e798897b90ea81203b50e62df Mon Sep 17 00:00:00 2001 From: Kyle Hazen <41054023+k1sauce@users.noreply.github.com> Date: Thu, 11 Apr 2024 16:47:04 -0700 Subject: [PATCH 092/142] Update subworkflow_tests.py fix typo --- nf_core/subworkflows/lint/subworkflow_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 02a429b661..cfae2d553c 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -20,7 +20,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): It verifies that the test directory exists and contains a ``main.nf.test`` a ``main.nf.test.snap`` and ``tags.yml``. - Additionally, hecks that all included components in test ``main.nf`` are specified in ``test.yml`` + Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ repo_dir = subworkflow.component_dir.parts[ From 941d087168007edfddfa927b3cec0594be57e0f9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 04:38:16 +0000 Subject: [PATCH 093/142] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d7fed0449d..9e227aa0b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.6 + rev: v0.3.7 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From c7361a0d24b7c5a4f5c063bba82d8f929cbc5a07 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 12 Apr 2024 04:39:06 +0000 Subject: [PATCH 094/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e758c6e60..e525ae4ecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ - Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) - Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From baf3519652d9256de4e13103de731d7c39c12a32 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 22:57:46 +0000 Subject: [PATCH 095/142] Update gitpod/workspace-base Docker digest to 0af257e --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index ca69f74f71..fd734381a5 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:674315a74b11e7b796040587d5e5ad000d755e0f14dd070704757e578be25fd9 +FROM gitpod/workspace-base@sha256:0af257e5e7f0340db76c905ca6acbe3d432567aafe53ffc0c91c369fc1183428 USER root From 3a7f8e6de2b74898844b75c592a9c03c0cab50af Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 17 Apr 2024 02:02:20 +0000 Subject: [PATCH 096/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e525ae4ecf..d7ba6a7a46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ - Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) +- Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 6fa38cbf6033554c4086fa491d83be8a63fe96c2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 19 Apr 2024 18:11:59 +0000 Subject: [PATCH 097/142] Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.1 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e227aa0b1..a90a732bfe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.7 + rev: v0.4.1 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 6313aa254715bafc97f001a8d0428ff2509cc6fd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 19 Apr 2024 02:07:21 +0000 Subject: [PATCH 098/142] Update GitHub Actions --- .github/actions/create-lint-wf/action.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/pytest.yml | 4 ++-- .github/workflows/sync.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 1741b934f2..27cb4dbc9f 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -77,7 +77,7 @@ runs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-lint-wf/log.txt diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 4ef448e5ec..3ee9686f94 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -158,7 +158,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: nf-core-log-file-${{ matrix.TEMPLATE }} path: create-test-lint-wf/artifact_files.tar diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 952181c044..88962a0bc7 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -75,7 +75,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-test-wf/log.txt diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 1303409c05..ea535394ea 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -143,7 +143,7 @@ jobs: fi - name: Upload coverage - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: coverage_${{ matrix.test }} path: .coverage @@ -177,7 +177,7 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4 + uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 2adb970f64..e54436af0e 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -86,7 +86,7 @@ jobs: - name: Upload sync log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: sync_log_${{ matrix.pipeline }} path: sync_log_${{ matrix.pipeline }}.txt diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index a8208b31ac..719689507f 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -60,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: linting-logs path: | From 360b044a656e39ab941b1645060a823c0b3d9cbc Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 20 Apr 2024 05:17:23 +0000 Subject: [PATCH 099/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7ba6a7a46..d4ae1e115d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) +- Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 8a781499192572072ce535e2b2346967ce911a4f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 13:32:25 +0000 Subject: [PATCH 100/142] chore(deps): update gitpod/workspace-base docker digest to 5aeb24f --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index fd734381a5..be4feab2c5 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:0af257e5e7f0340db76c905ca6acbe3d432567aafe53ffc0c91c369fc1183428 +FROM gitpod/workspace-base@sha256:5aeb24f9994e8226744a200b8316d5d429c0ac98e0b49d1c05ee7d376d8420c5 USER root From ced4255d9dcaebf0033bf2614676c16764d99d15 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 23 Apr 2024 16:07:39 +0000 Subject: [PATCH 101/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ae1e115d..0b4931a639 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) - Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) +- Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 9bfdcee731d7ea198e3631116a859db25625f9e1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 07:38:55 +0000 Subject: [PATCH 102/142] chore(deps): update github actions --- .github/actions/create-lint-wf/action.yml | 2 +- .github/workflows/changelog.yml | 2 +- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 4 ++-- .github/workflows/create-test-wf.yml | 4 ++-- .github/workflows/deploy-pypi.yml | 2 +- .github/workflows/fix-linting.yml | 2 +- .github/workflows/lint-code.yml | 2 +- .github/workflows/push_dockerhub_dev.yml | 2 +- .github/workflows/push_dockerhub_release.yml | 2 +- .github/workflows/pytest.yml | 10 +++++----- .github/workflows/rich-codex.yml | 2 +- .github/workflows/sync.yml | 6 +++--- .github/workflows/update_components_template.yml | 2 +- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- .../.github/workflows/fix-linting.yml | 2 +- .../pipeline-template/.github/workflows/linting.yml | 6 +++--- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 27cb4dbc9f..0bc5e432e7 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -77,7 +77,7 @@ runs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-lint-wf/log.txt diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index bed104e2d9..5318d2a0c4 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -19,7 +19,7 @@ jobs: ) steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 with: token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 996bdf2799..521f5d038f 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -48,7 +48,7 @@ jobs: export NXF_WORK=$(pwd) # Get the repo code - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository # Set up nf-core/tools diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 3ee9686f94..1634ade745 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -61,7 +61,7 @@ jobs: cd create-lint-wf-template export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: Set up Python 3.12 @@ -158,7 +158,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.TEMPLATE }} path: create-test-lint-wf/artifact_files.tar diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 88962a0bc7..d752e26713 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -48,7 +48,7 @@ jobs: cd create-test-wf export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: Set up Python 3.12 @@ -75,7 +75,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-test-wf/log.txt diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index ec94ddb88f..0afe47b751 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index b3f73c5a8c..b89dd44aa9 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 51a543c143..9a9eb31bc0 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -18,7 +18,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index af11a79990..360c863af6 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 0b3f381d86..1babf607d7 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index ea535394ea..0d51739bf9 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -61,7 +61,7 @@ jobs: name: Get test file matrix runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: List tests @@ -87,7 +87,7 @@ jobs: cd pytest export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: Set up Python ${{ needs.setup.outputs.python-version }} @@ -143,7 +143,7 @@ jobs: fi - name: Upload coverage - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: coverage_${{ matrix.test }} path: .coverage @@ -158,7 +158,7 @@ jobs: mkdir -p pytest cd pytest - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 env: @@ -177,7 +177,7 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d # v4 + uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index fc8af8a7bc..016f20ab0e 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index e54436af0e..43bab50bbf 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -44,10 +44,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out nf-core/tools - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} @@ -86,7 +86,7 @@ jobs: - name: Upload sync log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: sync_log_${{ matrix.pipeline }} path: sync_log_${{ matrix.pipeline }}.txt diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index c5413cd557..d3690af6e3 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 3880b2c4dc..9603a597a9 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index bc15ce42e8..1dc4be0113 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 719689507f..2e75daf1a9 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,7 +14,7 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 @@ -60,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: linting-logs path: | From 6f7a30982b2b563df1c870dd54becd0ccd5e4bed Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 10:28:34 +0000 Subject: [PATCH 103/142] Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a90a732bfe..5ff5afd864 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: alias: ec - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.9.0" + rev: "v1.10.0" hooks: - id: mypy additional_dependencies: From 8dc2ae6b56ab830d8154ac7735cf32e6cb8aacc3 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 25 Apr 2024 10:29:26 +0000 Subject: [PATCH 104/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b4931a639..08ef6ed61e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) - Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) +- Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From e77ed1b254bfacd063a5334d8ca47bdd18466a88 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 20:30:35 +0000 Subject: [PATCH 105/142] Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ff5afd864..d8f35e2cc5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.1 + rev: v0.4.2 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From aaabf5faee9097d9b9970597c8ad88d23a41e53e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 26 Apr 2024 08:51:13 +0200 Subject: [PATCH 106/142] Update nextflow.config --- nf_core/pipeline-template/nextflow.config | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2f9b349916..e7a9f8cf61 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -76,12 +76,11 @@ try { } // Load {{ name }} custom profiles from different institutions. -// Warning: Uncomment only if a pipeline-specific institutional config already exists on nf-core/configs! -// try { -// includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" -// } catch (Exception e) { -// System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") -// } +try { + includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" +} catch (Exception e) { + System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") +} {% endif -%} profiles { From f1ce816aa549aa4fbb48515030592fb549b28c4c Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 26 Apr 2024 08:57:55 +0200 Subject: [PATCH 107/142] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ef6ed61e..e5a9610453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) +- Permanently activated pipeline-specific institutional configs support for all pipelines ([#2936](https://github.com/nf-core/tools/pull/2936)) ### Linting From c83cdfa82ee526fbc4c46ce1ca9bf761b955e95e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 26 Apr 2024 08:59:17 +0200 Subject: [PATCH 108/142] Apply suggestions from code review --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a9610453..b57381da11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) -- Permanently activated pipeline-specific institutional configs support for all pipelines ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) ### Linting From 91d264b0406c515edf478e02493d264a7430f3c0 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 26 Apr 2024 07:11:18 +0000 Subject: [PATCH 109/142] [automated] Fix code linting --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b57381da11..e0ba31484f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) -- Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) ### Linting From 2d029dbb86ed9098881991df0f0264c8a75aed0c Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 26 Apr 2024 14:13:44 +0200 Subject: [PATCH 110/142] fix ruff linting (UP031) --- nf_core/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index 9d4decc424..8ffecc0f55 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -925,7 +925,7 @@ def rectify_raw_container_matches(self, raw_findings): if bool(container_definition) and bool(container_definition.group(1)): pattern = re.escape(container_definition.group(1)) # extract the quoted string(s) following the variable assignment - container_names = re.findall(r"%s\s*=\s*[\"\']([^\"\']+)[\"\']" % pattern, search_space) + container_names = re.findall(rf"{pattern}\s*=\s*[\"\']([^\"\']+)[\"\']", search_space) if bool(container_names): if isinstance(container_names, str): From 352c02cb18000118218793aa995d61d62606c8aa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:28:12 +0000 Subject: [PATCH 111/142] Update GitHub Actions --- .github/workflows/changelog.yml | 2 +- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/deploy-pypi.yml | 2 +- .github/workflows/fix-linting.yml | 2 +- .github/workflows/lint-code.yml | 2 +- .github/workflows/push_dockerhub_dev.yml | 2 +- .github/workflows/push_dockerhub_release.yml | 2 +- .github/workflows/pytest.yml | 8 ++++---- .github/workflows/rich-codex.yml | 2 +- .github/workflows/sync.yml | 4 ++-- .github/workflows/update_components_template.yml | 2 +- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- .../pipeline-template/.github/workflows/fix-linting.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 4 ++-- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 5318d2a0c4..cebcc854bc 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -19,7 +19,7 @@ jobs: ) steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 521f5d038f..03b9aa2411 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -48,7 +48,7 @@ jobs: export NXF_WORK=$(pwd) # Get the repo code - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository # Set up nf-core/tools diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 1634ade745..0de7287a57 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -61,7 +61,7 @@ jobs: cd create-lint-wf-template export NXF_WORK=$(pwd) - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index d752e26713..87cdf2e7bb 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -48,7 +48,7 @@ jobs: cd create-test-wf export NXF_WORK=$(pwd) - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 0afe47b751..1202891e4d 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index b89dd44aa9..4334871c4c 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 9a9eb31bc0..3bddd42d49 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -18,7 +18,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index 360c863af6..c613e13a2d 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 1babf607d7..5a076f6d3b 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0d51739bf9..03576fa310 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -61,7 +61,7 @@ jobs: name: Get test file matrix runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: List tests @@ -87,7 +87,7 @@ jobs: cd pytest export NXF_WORK=$(pwd) - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python ${{ needs.setup.outputs.python-version }} @@ -158,7 +158,7 @@ jobs: mkdir -p pytest cd pytest - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 env: @@ -177,7 +177,7 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4 + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 016f20ab0e..cd12b139d3 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 43bab50bbf..15802cdef2 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -44,10 +44,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out nf-core/tools - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index d3690af6e3..5ba513735e 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 9603a597a9..6b2547765d 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 1dc4be0113..18e6f9e158 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 2e75daf1a9..d1ecae12b7 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,7 +14,7 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 From 025f51db4dabefbebe573bd144ff46337dfae4c6 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 26 Apr 2024 20:05:13 +0000 Subject: [PATCH 112/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ef6ed61e..b618f947ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ - Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) +- Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 952ae662321b8085e24ed87bead870a1980dce27 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 15:22:44 +0200 Subject: [PATCH 113/142] Template: Add config profile 'wave' --- nf_core/pipeline-template/docs/usage.md | 2 ++ nf_core/pipeline-template/nextflow.config | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index c908d3d38c..b8305bf67f 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -163,6 +163,8 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) - `apptainer` - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) +- `wave` + - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above. - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2f9b349916..ad22abb5ae 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -176,6 +176,11 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } + wave { + wave.enabled = true + wave.strategy = 'conda,container' + wave.freeze = true + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } From 8e9b3e6880f4b5fd14189b6dfe20458f9f05e472 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 15:37:07 +0200 Subject: [PATCH 114/142] Move Wave above GitPod --- nf_core/pipeline-template/nextflow.config | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ad22abb5ae..b95a741299 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -171,16 +171,16 @@ profiles { shifter.enabled = false charliecloud.enabled = false } - gitpod { - executor.name = 'local' - executor.cpus = 4 - executor.memory = 8.GB - } wave { wave.enabled = true wave.strategy = 'conda,container' wave.freeze = true } + gitpod { + executor.name = 'local' + executor.cpus = 4 + executor.memory = 8.GB + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } From 8875f1009db9b0815da68b6b63aad8542bc7d559 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 15:57:48 +0200 Subject: [PATCH 115/142] Add 'singularity.ociAutoPull = true' --- nf_core/pipeline-template/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index b95a741299..ebe7a34081 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -175,6 +175,7 @@ profiles { wave.enabled = true wave.strategy = 'conda,container' wave.freeze = true + singularity.ociAutoPull = true } gitpod { executor.name = 'local' From 6dc46f747f1ddc0b4020ce38ea6417e2b0581e54 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 16:18:07 +0200 Subject: [PATCH 116/142] Harshil aligned and Maxime sorted Co-authored-by: Maxime U Garcia --- nf_core/pipeline-template/nextflow.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ebe7a34081..ea4e3b44be 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -172,10 +172,10 @@ profiles { charliecloud.enabled = false } wave { - wave.enabled = true - wave.strategy = 'conda,container' - wave.freeze = true singularity.ociAutoPull = true + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' } gitpod { executor.name = 'local' From ac5d1e8bcce95ebc1d5601b05ea9d80dd12f5dac Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 17:12:19 +0200 Subject: [PATCH 117/142] Added 'apptainer.ociAutoPull' --- nf_core/pipeline-template/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ea4e3b44be..2aa2c5914c 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -172,6 +172,7 @@ profiles { charliecloud.enabled = false } wave { + apptainer.ociAutoPull = true singularity.ociAutoPull = true wave.enabled = true wave.freeze = true From 76ed900bf28cdf237a24ffeeb82d17f7933ade99 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 17:18:07 +0200 Subject: [PATCH 118/142] Add minimum Nextflow version to the docs --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index b8305bf67f..d46dfca04c 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -164,7 +164,7 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - `apptainer` - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) - `wave` - - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above. + - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow ` 24.03.0-edge` or later). - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. From 90a107054112fecd156e2b7802d72f85c68be8d4 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Wed, 1 May 2024 14:08:22 -0400 Subject: [PATCH 119/142] Extend repo_full_name_from_remote function to accomodate paths that begin with ssh:// ftp[s]:// etc. --- nf_core/modules/modules_utils.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index ca8993483b..21169516bd 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -20,22 +20,20 @@ def repo_full_name_from_remote(remote_url: str) -> str: Extracts the path from the remote URL See https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS for the possible URL patterns """ - # Check whether we have a https or ssh url - if remote_url.startswith("https"): - path = urlparse(remote_url).path - # Remove the intial '/' - path = path[1:] - # Remove extension - path = os.path.splitext(path)[0] + + if remote_url.startswith(("https://", "http://", "ftps://", "ftp://", "ssh://")): + # Parse URL and remove the initial '/' + path = urlparse(remote_url).path.lstrip('/') + elif 'git@' in remote_url: + # Extract the part after 'git@' and parse it + path = urlparse(remote_url.split('git@')[-1]).path else: - # Remove the initial `git@`` - split_path: list = remote_url.split("@") - path = split_path[-1] if len(split_path) > 1 else split_path[0] - path = urlparse(path).path - # Remove extension - path = os.path.splitext(path)[0] - return path + path = urlparse(remote_url).path + + # Remove the file extension from the path + path, _ = os.path.splitext(path) + return path def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], List[NFCoreComponent]]: """ From bb4992090d2c2a97c5ccec16ebd850d7b4a34d0f Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Wed, 1 May 2024 14:17:26 -0400 Subject: [PATCH 120/142] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b618f947ba..0e38e319c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## v2.13.2dev +- Handle more complete list of possible git URL forms (ssh:// and ftp:// prefixes specifically) + ### Template - Remove fasta default from nextflow.config ([#2828](https://github.com/nf-core/tools/pull/2828)) From 453f9ea2b23d34780ef417e2ba4842e7f7b6e3d9 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Wed, 1 May 2024 14:24:06 -0400 Subject: [PATCH 121/142] Ruff formatting fix --- nf_core/modules/modules_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index 21169516bd..6796de41ec 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -23,10 +23,10 @@ def repo_full_name_from_remote(remote_url: str) -> str: if remote_url.startswith(("https://", "http://", "ftps://", "ftp://", "ssh://")): # Parse URL and remove the initial '/' - path = urlparse(remote_url).path.lstrip('/') - elif 'git@' in remote_url: + path = urlparse(remote_url).path.lstrip("/") + elif "git@" in remote_url: # Extract the part after 'git@' and parse it - path = urlparse(remote_url.split('git@')[-1]).path + path = urlparse(remote_url.split("git@")[-1]).path else: path = urlparse(remote_url).path @@ -35,6 +35,7 @@ def repo_full_name_from_remote(remote_url: str) -> str: return path + def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], List[NFCoreComponent]]: """ Make a list of all modules installed in this repository From ca79d543b3f4dac20b9480a67be31b727848d8b3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 00:22:59 +0000 Subject: [PATCH 122/142] chore(deps): update codecov/codecov-action digest to 5ecb98a --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 03576fa310..70b9cfd0a8 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -184,7 +184,7 @@ jobs: coverage report coverage xml - - uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # v4 + - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be # v4 with: files: coverage.xml env: From 3ad567e960f32c281c8630d002b1e350c60c8298 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 00:58:39 +0000 Subject: [PATCH 123/142] Update gitpod/workspace-base Docker digest to 124f2b8 --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index be4feab2c5..6afca0e479 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:5aeb24f9994e8226744a200b8316d5d429c0ac98e0b49d1c05ee7d376d8420c5 +FROM gitpod/workspace-base@sha256:124f2b8cbefe9b4abbb6a14538da8846770dde20b93f038d9551b6230aec1d1c USER root From 01d70a99101c1609d7934463b34dc646bee4d43e Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 1 May 2024 04:20:00 +0000 Subject: [PATCH 124/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b618f947ba..50c585879e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) +- Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From cc4e4165c1cca3fd842cc845d3d2963ad787f5f6 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 2 May 2024 11:09:33 +0200 Subject: [PATCH 125/142] More @drpatelh alignment to satisfy @mashehu --- nf_core/pipeline-template/nextflow.config | 136 +++++++++++----------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2aa2c5914c..931ff013a7 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -86,90 +86,90 @@ try { profiles { debug { - dumpHashes = true - process.beforeScript = 'echo $HOSTNAME' - cleanup = false + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + cleanup = false nextflow.enable.configProcessNamesValidation = true } conda { - conda.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - channels = ['conda-forge', 'bioconda', 'defaults'] - apptainer.enabled = false + conda.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + channels = ['conda-forge', 'bioconda', 'defaults'] + apptainer.enabled = false } mamba { - conda.enabled = true - conda.useMamba = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + conda.enabled = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } docker { - docker.enabled = true - conda.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false - docker.runOptions = '-u $(id -u):$(id -g)' + docker.enabled = true + conda.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + docker.runOptions = '-u $(id -u):$(id -g)' } arm { - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { - singularity.enabled = true - singularity.autoMounts = true - conda.enabled = false - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + singularity.enabled = true + singularity.autoMounts = true + conda.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } podman { - podman.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + podman.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } shifter { - shifter.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + shifter.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } charliecloud { - charliecloud.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - apptainer.enabled = false + charliecloud.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + apptainer.enabled = false } apptainer { - apptainer.enabled = true - apptainer.autoMounts = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + apptainer.enabled = true + apptainer.autoMounts = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false } wave { apptainer.ociAutoPull = true @@ -179,9 +179,9 @@ profiles { wave.strategy = 'conda,container' } gitpod { - executor.name = 'local' - executor.cpus = 4 - executor.memory = 8.GB + executor.name = 'local' + executor.cpus = 4 + executor.memory = 8.GB } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } From b63087e4bf960a032b67b0a9d382f69a451c4f3f Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 2 May 2024 11:52:44 +0200 Subject: [PATCH 126/142] fix(template): fix handling of multiple DOIs in manifest --- .../local/utils_nfcore_pipeline_pipeline/main.nf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index e02a113755..07cbb0bc56 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -243,8 +243,14 @@ def methodsDescriptionText(mqc_methods_yaml) { meta["manifest_map"] = workflow.manifest.toMap() // Pipeline DOI - meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" - meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + if (meta.manifest_map.doi) { + def temp_doi_ref = "" + String[] manifest_doi = meta.manifest_map.doi.split(",") + for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref}), " + meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length()-2) + } else { + meta["nodoi_text"] = "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + } // Tool references meta["tool_citations"] = "" From 13e5170296524e03f7b21543841b29848fc1e600 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 2 May 2024 12:04:46 +0200 Subject: [PATCH 127/142] Update nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf --- .../local/utils_nfcore_pipeline_pipeline/main.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 07cbb0bc56..479900d5a9 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -248,9 +248,8 @@ def methodsDescriptionText(mqc_methods_yaml) { String[] manifest_doi = meta.manifest_map.doi.split(",") for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref}), " meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length()-2) - } else { - meta["nodoi_text"] = "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " - } + } else meta["doi_text"] = "" + meta["nodoi_text"] = meta.manifest_map.doi ? "" : "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " // Tool references meta["tool_citations"] = "" From d6e1b102175feaf96dfaa6cda0cfb74629cb0195 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 2 May 2024 12:17:57 +0200 Subject: [PATCH 128/142] fix(collectfile): sort true for methods_description_mqc.yaml --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 8cdff1baa3..de0f21fe38 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -75,7 +75,7 @@ workflow {{ short_name|upper }} { ch_multiqc_files = ch_multiqc_files.mix( ch_methods_description.collectFile( name: 'methods_description_mqc.yaml', - sort: false + sort: true ) ) From 7786e255ea99d34c34f0d34b63f0b1a039f0655a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 2 May 2024 10:19:31 +0000 Subject: [PATCH 129/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0cca60b0d..9b27f8b8c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) +- fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From f382f980a1be496e96726a8293ae9c031247f925 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 2 May 2024 18:23:42 +0200 Subject: [PATCH 130/142] handles doi.org resolver + comments --- .../local/utils_nfcore_pipeline_pipeline/main.nf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 479900d5a9..a4bfb9f8be 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -244,10 +244,13 @@ def methodsDescriptionText(mqc_methods_yaml) { // Pipeline DOI if (meta.manifest_map.doi) { + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list def temp_doi_ref = "" - String[] manifest_doi = meta.manifest_map.doi.split(",") - for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref}), " - meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length()-2) + String[] manifest_doi = meta.manifest_map.doi.tokenize(",") + for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref.replace("https://doi.org/", "").replace(" ", "")}), " + meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length() - 2) } else meta["doi_text"] = "" meta["nodoi_text"] = meta.manifest_map.doi ? "" : "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " From fa0b60d642b964d7a32249d0838f6d39a0b8d9af Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 2 May 2024 18:24:41 +0200 Subject: [PATCH 131/142] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0cca60b0d..d2a2bcf042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) - Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Handles multiple DOIs + doi.org resolver from manifest.doi ([#2946](https://github.com/nf-core/tools/pull/2946)) ### Linting From c5d55b8f4a5b4c2107ef6c4be0f660cd24e4796a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 03:48:04 +0000 Subject: [PATCH 132/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b618f947ba..439c38769d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) +- Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 80fcd09575dc2b4972c792622543d2f1e9f39d30 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 3 May 2024 08:03:13 +0200 Subject: [PATCH 133/142] update components to fix CI testing --- nf_core/pipeline-template/modules.json | 4 ++-- nf_core/pipeline-template/modules/nf-core/fastqc/main.nf | 6 ++++++ .../subworkflows/nf-core/utils_nfcore_pipeline/main.nf | 8 +++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 38aa622063..9137c59675 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,7 +7,7 @@ "nf-core": { "fastqc": { "branch": "master", - "git_sha": "f4ae1d942bd50c5c0b9bd2de1393ce38315ba57c", + "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", "installed_by": ["modules"] }, "multiqc": { @@ -26,7 +26,7 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", "installed_by": ["subworkflows"] }, "utils_nfvalidation_plugin": { diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 9e19a74c56..d79f1c862d 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -25,6 +25,11 @@ process FASTQC { def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + + def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') + // FastQC memory value allowed range (100 - 10000) + def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) + """ printf "%s %s\\n" $rename_to | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name @@ -33,6 +38,7 @@ process FASTQC { fastqc \\ $args \\ --threads $task.cpus \\ + --memory $fastqc_memory \\ $renamed_files cat <<-END_VERSIONS > versions.yml diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index a8b55d6fe1..14558c3927 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -65,9 +65,15 @@ def checkProfileProvided(nextflow_cli_args) { // Citation string for pipeline // def workflowCitation() { + def temp_doi_ref = "" + String[] manifest_doi = workflow.manifest.doi.tokenize(",") + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + for (String doi_ref: manifest_doi) temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + "* The pipeline\n" + - " ${workflow.manifest.doi}\n\n" + + temp_doi_ref + "\n" + "* The nf-core framework\n" + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + "* Software dependencies\n" + From 13b1f10f796729acd8c680ca7d16516ac56827a1 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 06:04:58 +0000 Subject: [PATCH 134/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7ba6a7a46..015a1d0825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) +- update components to fix CI testing ([#2949](https://github.com/nf-core/tools/pull/2949)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 32ac3837e94d52f1ce0331cead78f86ef5761dbf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 06:14:02 +0000 Subject: [PATCH 135/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e38d8bb39..5e10a63824 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,13 +58,13 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) -- update components to fix CI testing ([#2949](https://github.com/nf-core/tools/pull/2949)) - Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) - Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) +- Template: Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 0adbe37e19ae36e3488b0d592e08b08fbaf62d1f Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 3 May 2024 08:16:48 +0200 Subject: [PATCH 136/142] fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e10a63824..40450bc51d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) - Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) ### Linting @@ -64,7 +65,6 @@ - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) - Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) -- Template: Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From e674d3cd5ac416d74a39b7f67eced22922f6516f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 3 May 2024 09:39:00 +0200 Subject: [PATCH 137/142] Template config: `conda.channels`, not `channels` --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index fb3f08c62e..936e5e80d7 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -97,7 +97,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false - channels = ['conda-forge', 'bioconda', 'defaults'] + conda.channels = ['conda-forge', 'bioconda', 'defaults'] apptainer.enabled = false } mamba { From c7ea164792157e04b2ef55a01a672d1f8eb97c38 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 07:40:04 +0000 Subject: [PATCH 138/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e832326fb..d43c0d8838 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) - Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Template config: `conda.channels`, not `channels` ([#2950](https://github.com/nf-core/tools/pull/2950)) ### Linting From d0c17ca718c942c75c67bc7f08e0d61a97b11b57 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 10:39:53 +0000 Subject: [PATCH 139/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e38e319c0..6ee191a4b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) +- Components: Handle extended set of git path prefixes ([#2945](https://github.com/nf-core/tools/pull/2945)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From aa6310939a7edd81a23e894182c2fa0ee9a902fa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 23:27:47 +0000 Subject: [PATCH 140/142] Update python:3.12-slim Docker digest to 2be8dad --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c801563667..ae3a4e1a3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:541d45d3d675fb8197f534525a671e2f8d66c882b89491f9dda271f4f94dcd06 +FROM python:3.12-slim@sha256:2be8daddbb82756f7d1f2c7ece706aadcb284bf6ab6d769ea695cc3ed6016743 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From ffa64950ea26064ac9f0cbc8a839ddc74d53373b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 00:58:27 +0000 Subject: [PATCH 141/142] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8f35e2cc5..887cbe027c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.2 + rev: v0.4.3 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 7cfcdd69eaf6a23180d9e4e45ae88e6989dbdf2e Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 4 May 2024 00:59:14 +0000 Subject: [PATCH 142/142] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a02e9a929..00f0c0ed0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ - Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) - fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 ([#2951](https://github.com/nf-core/tools/pull/2951)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29]