From 998a22c9a98d5642d5df22dc816971c7d9acc4aa Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 12:44:12 +0100 Subject: [PATCH 01/22] add ruff formatter and linter --- .github/workflows/create-lint-wf.yml | 8 +++++ .github/workflows/fix-linting.yml | 7 +++++ .github/workflows/lint-code.yml | 39 ++++++++++++++++++++++++ .gitpod.yml | 1 + .pre-commit-config.yaml | 6 ++++ CHANGELOG.md | 2 ++ README.md | 1 + nf_core/pipeline-template/pyproject.toml | 8 +++++ pyproject.toml | 8 +++++ requirements-dev.txt | 1 + 10 files changed, 81 insertions(+) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 57dbe86d65..bd3c65859c 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -55,6 +55,11 @@ jobs: - name: Install Prettier and editorconfig-checker run: npm install -g prettier editorconfig-checker + - name: Install ruff + run: | + python -m pip install --upgrade pip + pip install ruff + # Build a pipeline from the template - name: nf-core create run: | @@ -76,6 +81,9 @@ jobs: - name: Run Prettier --check run: prettier --check create-lint-wf/nf-core-testpipeline + - name: Run Ruff check + run: ruff check create-lint-wf/nf-core-testpipeline + - name: Run ECLint check run: editorconfig-checker -exclude README.md $(find nf-core-testpipeline/.* -type f | grep -v '.git\|.py\|md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') working-directory: create-lint-wf diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 4184bc5e59..d83f96e363 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -50,6 +50,13 @@ jobs: isortVersion: "latest" requirementsFiles: "requirements.txt requirements-dev.txt" + - name: Install Ruff + run: | + python -m pip install --upgrade pip + pip install ruff + - name: Run Ruff + run: ruff format --fix . + - name: Commit & push changes run: | git config user.email "core@nf-co.re" diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 23972c56f6..8170ebea09 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -74,6 +74,45 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} allow-repeats: false + Ruff: + runs-on: ["self-hosted"] + steps: + - name: Check out source-code repository + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Install Ruff + run: | + python -m pip install --upgrade pip + pip install ruff + - name: Run Ruff + run: ruff check . + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Python linting (`ruff`) is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install [`ruff`](https://github.com/astral-sh/ruff): `pip install ruff` + * Fix formatting errors in your pipeline: `ruff formatter --fix .` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up Ruff in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false + isort: runs-on: ["self-hosted"] steps: diff --git a/.gitpod.yml b/.gitpod.yml index 3c8b6b5303..724d7e7735 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -21,3 +21,4 @@ vscode: # - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code + - - charliermarsh.ruff # Code linter Ruff diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad23a3c895..ebf41a38ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,10 @@ repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.9 + hooks: + - id: ruff-format # formatter + - id: ruff # linter + args: [--fix] - repo: https://github.com/psf/black rev: 23.1.0 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a252cbf67..37fb75845b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ ### General +- Add Ruff linter and formatter + # [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20] ### Template diff --git a/README.md b/README.md index 653f5295ea..edf3037898 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![code style: prettier](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://github.com/prettier/prettier) [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) +[![code style: Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v1.json)](https://github.com/charliermarsh/ruff) [![install with Bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/recipes/nf-core/README.html) [![install with PyPI](https://img.shields.io/badge/install%20with-PyPI-blue.svg)](https://pypi.org/project/nf-core/) diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 0d62beb6f9..d251800653 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -8,3 +8,11 @@ target_version = ["py37", "py38", "py39", "py310"] profile = "black" known_first_party = ["nf_core"] multi_line_output = 3 + +[tool.ruff] +line-length = 120 +target-version = "py38" +ignore = ["E501"] + +[tool.ruff.isort] +known-first-party = ["nf_core"] diff --git a/pyproject.toml b/pyproject.toml index 2380073107..f56a0b1196 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,3 +20,11 @@ norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".githu profile = "black" known_first_party = ["nf_core"] multi_line_output = 3 + +[tool.ruff] +line-length = 120 +target-version = "py38" +ignore = ["E501"] + +[tool.ruff.isort] +known-first-party = ["nf_core"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 13dba6f30d..f7e6c0a68f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,3 +13,4 @@ types-requests types-jsonschema types-Markdown types-setuptools +ruff From 9fdcf039f539902a138fb2f84c0a82140af38cf6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 13:30:03 +0100 Subject: [PATCH 02/22] remove isort --- .github/workflows/fix-linting.yml | 5 ----- .github/workflows/lint-code.yml | 16 ---------------- .pre-commit-config.yaml | 4 ---- README.md | 1 - nf_core/pipeline-template/pyproject.toml | 5 ----- pyproject.toml | 5 ----- requirements-dev.txt | 1 - 7 files changed, 37 deletions(-) diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index d83f96e363..e3789c85bd 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -44,11 +44,6 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.11 - - name: python-isort - uses: isort/isort-action@v1.0.0 - with: - isortVersion: "latest" - requirementsFiles: "requirements.txt requirements-dev.txt" - name: Install Ruff run: | diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 8170ebea09..4461a97fd8 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -113,22 +113,6 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} allow-repeats: false - isort: - runs-on: ["self-hosted"] - steps: - - name: Check out source-code repository - uses: actions/checkout@v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@v4 - with: - python-version: 3.11 - - name: python-isort - uses: isort/isort-action@v1.1.0 - with: - isortVersion: "latest" - requirementsFiles: "requirements.txt requirements-dev.txt" - static-type-check: runs-on: ["self-hosted"] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ebf41a38ea..00b964430e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,10 +9,6 @@ repos: rev: 23.1.0 hooks: - id: black - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - repo: https://github.com/pre-commit/mirrors-prettier rev: "v2.7.1" hooks: diff --git a/README.md b/README.md index edf3037898..a98d57d66a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ [![codecov](https://codecov.io/gh/nf-core/tools/branch/master/graph/badge.svg)](https://codecov.io/gh/nf-core/tools) [![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![code style: prettier](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://github.com/prettier/prettier) -[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) [![code style: Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v1.json)](https://github.com/charliermarsh/ruff) [![install with Bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/recipes/nf-core/README.html) diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index d251800653..ce3c302bf3 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -4,11 +4,6 @@ line-length = 120 target_version = ["py37", "py38", "py39", "py310"] -[tool.isort] -profile = "black" -known_first_party = ["nf_core"] -multi_line_output = 3 - [tool.ruff] line-length = 120 target-version = "py38" diff --git a/pyproject.toml b/pyproject.toml index f56a0b1196..5cdf0e9857 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,11 +16,6 @@ markers = [ testpaths = ["tests"] norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".github", "nf_core", "docs"] -[tool.isort] -profile = "black" -known_first_party = ["nf_core"] -multi_line_output = 3 - [tool.ruff] line-length = 120 target-version = "py38" diff --git a/requirements-dev.txt b/requirements-dev.txt index f7e6c0a68f..e03db2484d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,4 @@ black -isort myst_parser pytest-cov pytest-datafiles From 418f0be67866489ad059b1c16d537d184a52c168 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 13:36:54 +0100 Subject: [PATCH 03/22] remove python black --- .github/workflows/fix-linting.yml | 6 ---- .github/workflows/lint-code.yml | 30 ------------------- .pre-commit-config.yaml | 4 --- README.md | 1 - nf_core/gitpod/gitpod.Dockerfile | 18 +++++------ .../.github/workflows/linting.yml | 27 +++++++++++------ nf_core/pipeline-template/pyproject.toml | 6 +--- pyproject.toml | 4 --- requirements-dev.txt | 1 - 9 files changed, 28 insertions(+), 69 deletions(-) diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index e3789c85bd..b6ed08e885 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -34,12 +34,6 @@ jobs: - name: Run 'prettier --write' run: prettier --write ${GITHUB_WORKSPACE} - - name: Run Black - uses: psf/black@stable - with: - # Override to remove the default --check flag so that we make changes - options: "--color" - - name: Set up Python 3.11 uses: actions/setup-python@v4 with: diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 4461a97fd8..e2d9d3bb92 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -44,36 +44,6 @@ jobs: - name: Run Prettier --check run: prettier --check ${GITHUB_WORKSPACE} - PythonBlack: - runs-on: ["self-hosted"] - steps: - - uses: actions/checkout@v4 - - - name: Check code lints with Black - uses: psf/black@stable - - # If the above check failed, post a comment on the PR explaining the failure - - name: Post PR comment - if: failure() - uses: mshick/add-pr-comment@v1 - with: - message: | - ## Python linting (`black`) is failing - - To keep the code consistent with lots of contributors, we run automated code consistency checks. - To fix this CI test, please run: - - * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` - * Fix formatting errors in your pipeline: `black .` - - Once you push these changes the test should pass, and you can hide this comment :+1: - - We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! - - Thanks again for your contribution! - repo-token: ${{ secrets.GITHUB_TOKEN }} - allow-repeats: false - Ruff: runs-on: ["self-hosted"] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00b964430e..90d8523d8c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,10 +5,6 @@ repos: - id: ruff-format # formatter - id: ruff # linter args: [--fix] - - repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - repo: https://github.com/pre-commit/mirrors-prettier rev: "v2.7.1" hooks: diff --git a/README.md b/README.md index a98d57d66a..3a9b31187a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![Python tests](https://github.com/nf-core/tools/workflows/Python%20tests/badge.svg?branch=master&event=push)](https://github.com/nf-core/tools/actions?query=workflow%3A%22Python+tests%22+branch%3Amaster) [![codecov](https://codecov.io/gh/nf-core/tools/branch/master/graph/badge.svg)](https://codecov.io/gh/nf-core/tools) -[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![code style: prettier](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://github.com/prettier/prettier) [![code style: Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v1.json)](https://github.com/charliermarsh/ruff) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index e721f210d0..de1086410b 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -47,15 +47,15 @@ RUN conda config --add channels defaults && \ conda config --add channels conda-forge && \ conda config --set channel_priority strict && \ conda install --quiet --yes --name base \ - mamba \ - nextflow \ - nf-core \ - nf-test \ - black \ - prettier \ - pre-commit \ - openjdk \ - pytest-workflow && \ + mamba \ + nextflow \ + nf-core \ + nf-test \ + prettier \ + pre-commit \ + ruff \ + openjdk \ + pytest-workflow && \ conda clean --all --force-pkgs-dirs --yes # Update Nextflow diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 94aa5278be..188c3b3aeb 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -37,13 +37,22 @@ jobs: - name: Run Prettier --check run: prettier --check ${GITHUB_WORKSPACE} - PythonBlack: - runs-on: ubuntu-latest + Ruff: + runs-on: ["self-hosted"] steps: - - uses: actions/checkout@v4 + - name: Check out source-code repository + uses: actions/checkout@v4 - - name: Check code lints with Black - uses: psf/black@stable + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Install Ruff + run: | + python -m pip install --upgrade pip + pip install ruff + - name: Run Ruff + run: ruff check . # If the above check failed, post a comment on the PR explaining the failure - name: Post PR comment @@ -51,17 +60,17 @@ jobs: uses: mshick/add-pr-comment@v1 with: message: | - ## Python linting (`black`) is failing + ## Python linting (`ruff`) is failing To keep the code consistent with lots of contributors, we run automated code consistency checks. To fix this CI test, please run: - * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` - * Fix formatting errors in your pipeline: `black .` + * Install [`ruff`](https://github.com/astral-sh/ruff): `pip install ruff` + * Fix formatting errors in your pipeline: `ruff formatter --fix .` Once you push these changes the test should pass, and you can hide this comment :+1: - We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + We highly recommend setting up Ruff in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! Thanks again for your contribution! repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index ce3c302bf3..773e1eae58 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -1,9 +1,5 @@ -# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Black. +# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Ruff. # Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. -[tool.black] -line-length = 120 -target_version = ["py37", "py38", "py39", "py310"] - [tool.ruff] line-length = 120 target-version = "py38" diff --git a/pyproject.toml b/pyproject.toml index 5cdf0e9857..a4b5829c93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,10 +5,6 @@ requires = [ "wheel" ] -[tool.black] -line-length = 120 -target_version = ["py37", "py38", "py39", "py310"] - [tool.pytest.ini_options] markers = [ "datafiles: load datafiles" diff --git a/requirements-dev.txt b/requirements-dev.txt index e03db2484d..6e8cff2c5e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,3 @@ -black myst_parser pytest-cov pytest-datafiles From 2ca9b19cdb9140662888d691ba3cf4db3d288680 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 13:38:21 +0100 Subject: [PATCH 04/22] remove pyupgrade --- .pre-commit-config.yaml | 5 ----- requirements-dev.txt | 1 - 2 files changed, 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 90d8523d8c..7d84e37645 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,11 +9,6 @@ repos: rev: "v2.7.1" hooks: - id: prettier - - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 - hooks: - - id: pyupgrade - args: [--py38-plus] - repo: https://github.com/pre-commit/mirrors-mypy rev: "v1.7.1" # Use the sha / tag you want to point at hooks: diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e8cff2c5e..6d890f0e1f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,6 @@ Sphinx sphinx-rtd-theme mypy types-PyYAML -pyupgrade types-requests types-jsonschema types-Markdown From c25c5bee38aeb63054820634cfee28f341d6fbbc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 13:38:44 +0100 Subject: [PATCH 05/22] update CONTRIBUTING.md --- .github/CONTRIBUTING.md | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 75da414db6..b086b0d15e 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -35,45 +35,28 @@ pip install -e . ## Code formatting -### Black +### Ruff -All Python code in nf-core/tools must be passed through the [Black Python code formatter](https://black.readthedocs.io/en/stable/). +All Python code in nf-core/tools must be passed through the [Ruff code linter and formatter](https://github.com/astral-sh/ruff). This ensures a harmonised code formatting style throughout the package, from all contributors. -You can run Black on the command line (it's included in `requirements-dev.txt`) - eg. to run recursively on the whole repository: +You can run Ruff on the command line (it's included in `requirements-dev.txt`) - eg. to run recursively on the whole repository: ```bash -black . +ruff format . ``` -Alternatively, Black has [integrations for most common editors](https://black.readthedocs.io/en/stable/editor_integration.html) +Alternatively, Black has [integrations for most common editors](https://github.com/astral-sh/ruff-lsp) and VSCode(https://github.com/astral-sh/ruff-vscode) to automatically format code when you hit save. -You can also set it up to run when you [make a commit](https://black.readthedocs.io/en/stable/version_control_integration.html). There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if -any code does not adhere to Black formatting. +any code does not adhere to Ruff formatting. -### isort - -All Python code must also be passed through [isort](https://pycqa.github.io/isort/index.html). -This ensures a harmonised imports throughout the package, from all contributors. - -To run isort on the command line recursively on the whole repository you can use: - -```bash -isort . -``` - -isort also has [plugins for most common editors](https://github.com/pycqa/isort/wiki/isort-Plugins) -to automatically format code when you hit save. -Or [version control integration](https://pycqa.github.io/isort/docs/configuration/pre-commit.html) to set it up to run when you make a commit. - -There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if -any code does not adhere to isort formatting. +Ruff has been adopted for linting and formatting in replacement of Black, isort (for imports) and pyupgrade. It also includes Flake8. ### pre-commit hooks -This repository comes with [pre-commit](https://pre-commit.com/) hooks for black, isort and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again. +This repository comes with [pre-commit](https://pre-commit.com/) hooks for ruff and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again. You can use the pre-commit hooks if you like, but you don't have to. The CI on Github will run the same checks as the tools installed with pre-commit. If the pre-commit checks pass, then the same checks in the CI will pass, too. From e40f7b966d802c4919d732e3df70767e3cd23069 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 14:29:45 +0100 Subject: [PATCH 06/22] select import sorting and run first linter and then formatter --- .github/workflows/fix-linting.yml | 4 +++- .github/workflows/lint-code.yml | 2 +- .pre-commit-config.yaml | 4 ++-- nf_core/pipeline-template/.github/workflows/linting.yml | 2 +- nf_core/pipeline-template/pyproject.toml | 1 + pyproject.toml | 1 + 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index b6ed08e885..ad3383097a 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -44,7 +44,9 @@ jobs: python -m pip install --upgrade pip pip install ruff - name: Run Ruff - run: ruff format --fix . + run: | + ruff check --fix . + ruff format . - name: Commit & push changes run: | diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index e2d9d3bb92..d99241470e 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -73,7 +73,7 @@ jobs: To fix this CI test, please run: * Install [`ruff`](https://github.com/astral-sh/ruff): `pip install ruff` - * Fix formatting errors in your pipeline: `ruff formatter --fix .` + * Fix formatting errors in your pipeline: `ruff check --fix .` and `ruff format .` Once you push these changes the test should pass, and you can hide this comment :+1: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d84e37645..6391761fa1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,9 +2,9 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.9 hooks: - - id: ruff-format # formatter - id: ruff # linter - args: [--fix] + args: [--fix, --exit-non-zero-on-fix] # sort imports and fix + - id: ruff-format # formatter - repo: https://github.com/pre-commit/mirrors-prettier rev: "v2.7.1" hooks: diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 188c3b3aeb..0911e9a7f6 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -66,7 +66,7 @@ jobs: To fix this CI test, please run: * Install [`ruff`](https://github.com/astral-sh/ruff): `pip install ruff` - * Fix formatting errors in your pipeline: `ruff formatter --fix .` + * Fix formatting errors in your pipeline: `ruff check --fix .` and `ruff format .` Once you push these changes the test should pass, and you can hide this comment :+1: diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 773e1eae58..480ee479ca 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -4,6 +4,7 @@ line-length = 120 target-version = "py38" ignore = ["E501"] +select = ["I", "E4", "E7", "E9", "F"] [tool.ruff.isort] known-first-party = ["nf_core"] diff --git a/pyproject.toml b/pyproject.toml index a4b5829c93..7f1bd692ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".githu line-length = 120 target-version = "py38" ignore = ["E501"] +select = ["I", "E4", "E7", "E9", "F"] [tool.ruff.isort] known-first-party = ["nf_core"] From df3d3a1a72611e0aee70ceb785769c0812449a98 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 14:43:40 +0100 Subject: [PATCH 07/22] ignore import checks in __init__.py files --- nf_core/pipeline-template/pyproject.toml | 3 +++ pyproject.toml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 480ee479ca..be5c608791 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -8,3 +8,6 @@ select = ["I", "E4", "E7", "E9", "F"] [tool.ruff.isort] known-first-party = ["nf_core"] + +[tool.ruff.per-file-ignores] +"__init__.py" = ["E402", "F401"] diff --git a/pyproject.toml b/pyproject.toml index 7f1bd692ba..7c3029e79c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,3 +20,6 @@ select = ["I", "E4", "E7", "E9", "F"] [tool.ruff.isort] known-first-party = ["nf_core"] + +[tool.ruff.per-file-ignores] +"__init__.py" = ["E402", "F401"] From 14ca61837a554b09264ca9455960fac3c95c28d0 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 15:03:19 +0100 Subject: [PATCH 08/22] run ruff check --fix --- nf_core/bump_version.py | 4 ++-- nf_core/components/components_utils.py | 2 +- nf_core/download.py | 8 ++++---- nf_core/lint/actions_ci.py | 3 +-- nf_core/lint/system_exit.py | 2 +- nf_core/lint/version_consistency.py | 7 ++++--- nf_core/modules/bump_versions.py | 3 +-- nf_core/modules/lint/main_nf.py | 8 ++++---- nf_core/modules/modules_repo.py | 2 -- nf_core/modules/modules_utils.py | 2 +- nf_core/params_file.py | 3 --- nf_core/refgenie.py | 1 - nf_core/schema.py | 2 +- nf_core/subworkflows/lint/meta_yml.py | 4 ++-- nf_core/synced_repo.py | 1 - tests/modules/bump_versions.py | 1 - tests/modules/create.py | 1 - tests/subworkflows/create.py | 1 - tests/subworkflows/lint.py | 1 - tests/subworkflows/remove.py | 4 +--- tests/subworkflows/update.py | 1 - tests/test_download.py | 6 ++---- tests/test_launch.py | 26 +++++++++++++------------- tests/test_lint.py | 2 +- tests/test_params_file.py | 2 +- tests/test_refgenie.py | 2 -- tests/test_schema.py | 4 ++-- tests/test_test_utils.py | 3 --- tests/utils.py | 4 +--- 29 files changed, 43 insertions(+), 67 deletions(-) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 40c8f8984f..b8f19ffb65 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -68,7 +68,7 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: [ ( f"/releases/tag/{current_version}", - f"/tree/dev", + "/tree/dev", ) ], ) @@ -78,7 +78,7 @@ def bump_pipeline_version(pipeline_obj: Pipeline, new_version: str) -> None: pipeline_obj, [ ( - f"/tree/dev", + "/tree/dev", f"/releases/tag/{multiqc_new_version}", ) ], diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index eec533ce60..69f9452d5e 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -53,7 +53,7 @@ def get_repo_info(directory: str, use_prompt: Optional[bool] = True) -> Tuple[st raise UserWarning("Repository type could not be established") # Check if it's a valid answer - if not repo_type in ["pipeline", "modules"]: + if repo_type not in ["pipeline", "modules"]: raise UserWarning(f"Invalid repository type: '{repo_type}'") # Check for org if modules repo diff --git a/nf_core/download.py b/nf_core/download.py index 08bef935ba..1b05c266ec 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -551,7 +551,7 @@ def read_remote_containers(self): self.containers_remote = sorted(list(set(self.containers_remote))) except (FileNotFoundError, LookupError) as e: log.error(f"[red]Issue with reading the specified remote $NXF_SINGULARITY_CACHE index:[/]\n{e}\n") - if stderr.is_interactive and rich.prompt.Confirm.ask(f"[blue]Specify a new index file and try again?"): + if stderr.is_interactive and rich.prompt.Confirm.ask("[blue]Specify a new index file and try again?"): self.container_cache_index = None # reset chosen path to index file. self.prompt_singularity_cachedir_remote() else: @@ -853,7 +853,7 @@ def rectify_raw_container_matches(self, raw_findings): ['https://depot.galaxyproject.org/singularity/scanpy:1.7.2--pyhdfd78af_0', 'biocontainers/scanpy:1.7.2--pyhdfd78af_0'] """ container_value_defs = [ - capture for _, capture in container_value_defs[:] if not capture in ["singularity", "apptainer"] + capture for _, capture in container_value_defs[:] if capture not in ["singularity", "apptainer"] ] """ @@ -1066,7 +1066,7 @@ def get_singularity_images(self, current_revision=""): self.singularity_pull_image(*container, library, progress) # Pulling the image was successful, no ContainerError was raised, break the library loop break - except ContainerError.ImageExists as e: + except ContainerError.ImageExists: # Pulling not required break except ContainerError.RegistryNotFound as e: @@ -1085,7 +1085,7 @@ def get_singularity_images(self, current_revision=""): break # there no point in trying other registries if absolute URI was specified. else: continue - except ContainerError.InvalidTag as e: + except ContainerError.InvalidTag: # Try other registries continue except ContainerError.OtherError as e: diff --git a/nf_core/lint/actions_ci.py b/nf_core/lint/actions_ci.py index e669eceb8c..1c0a505fa2 100644 --- a/nf_core/lint/actions_ci.py +++ b/nf_core/lint/actions_ci.py @@ -1,5 +1,4 @@ import os -import re import yaml @@ -62,7 +61,7 @@ def actions_ci(self): if not ( pr_subtree is None or ("branches" in pr_subtree and "dev" in pr_subtree["branches"]) - or ("ignore_branches" in pr_subtree and not "dev" in pr_subtree["ignore_branches"]) + or ("ignore_branches" in pr_subtree and "dev" not in pr_subtree["ignore_branches"]) ): raise AssertionError() if "published" not in ciwf[True]["release"]["types"]: diff --git a/nf_core/lint/system_exit.py b/nf_core/lint/system_exit.py index 56a526d97b..10c8d0bb4e 100644 --- a/nf_core/lint/system_exit.py +++ b/nf_core/lint/system_exit.py @@ -26,7 +26,7 @@ def system_exit(self): try: with file.open() as fh: for i, l in enumerate(fh.readlines(), start=1): - if "System.exit" in l and not "System.exit(0)" in l: + if "System.exit" in l and "System.exit(0)" not in l: warned.append(f"`System.exit` in {file.name}: _{l.strip()}_ [line {i}]") except FileNotFoundError: log.debug(f"Could not open file {file.name} in system_exit lint test") diff --git a/nf_core/lint/version_consistency.py b/nf_core/lint/version_consistency.py index fa5b50de01..4037bc01e4 100644 --- a/nf_core/lint/version_consistency.py +++ b/nf_core/lint/version_consistency.py @@ -31,7 +31,7 @@ def version_consistency(self): versions["manifest.version"] = self.nf_config.get("manifest.version", "").strip(" '\"") # Get version from the docker tag - if self.nf_config.get("process.container", "") and not ":" in self.nf_config.get("process.container", ""): + if self.nf_config.get("process.container", "") and ":" not in self.nf_config.get("process.container", ""): failed.append(f"Docker slug seems not to have a version tag: {self.nf_config.get('process.container', '')}") # Get config container tag (if set; one container per workflow) @@ -53,8 +53,9 @@ def version_consistency(self): # Check if they are consistent if len(set(versions.values())) != 1: failed.append( - "The versioning is not consistent between container, release tag " - "and config. Found {}".format(", ".join(["{} = {}".format(k, v) for k, v in versions.items()])) + "The versioning is not consistent between container, release tag " "and config. Found {}".format( + ", ".join(["{} = {}".format(k, v) for k, v in versions.items()]) + ) ) passed.append("Version tags are numeric and consistent between container, release tag and config.") diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 25259f1a16..b160466bb3 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -24,9 +24,8 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand from nf_core.components.nfcore_component import NFCoreComponent -from nf_core.utils import custom_yaml_dumper +from nf_core.utils import custom_yaml_dumper, rich_force_colors from nf_core.utils import plural_s as _s -from nf_core.utils import rich_force_colors log = logging.getLogger(__name__) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 56a9e99925..4d06589ced 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -263,7 +263,7 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): self.passed.append( ( "deprecated_enable_conda", - f"Deprecated parameter 'params.enable_conda' correctly not found in the conda definition", + "Deprecated parameter 'params.enable_conda' correctly not found in the conda definition", self.main_nf, ) ) @@ -271,7 +271,7 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): self.failed.append( ( "deprecated_enable_conda", - f"Found deprecated parameter 'params.enable_conda' in the conda definition", + "Found deprecated parameter 'params.enable_conda' in the conda definition", self.main_nf, ) ) @@ -308,7 +308,7 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): ) ) else: - self.passed.append(("container_links", f"Container prefix is correct", self.main_nf)) + self.passed.append(("container_links", "Container prefix is correct", self.main_nf)) # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04) # If so, add quay.io as default container prefix @@ -576,7 +576,7 @@ def _parse_output(self, line): output = [] if "meta" in line: output.append("meta") - if not "emit:" in line: + if "emit:" not in line: self.failed.append(("missing_emit", f"Missing emit statement: {line.strip()}", self.main_nf)) else: output.append(line.split("emit:")[1].strip()) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 152ed7b0c0..204c20fd71 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -1,8 +1,6 @@ -import filecmp import logging import os import shutil -from pathlib import Path import git import rich diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index 3ae01e9eef..ecc3e1605b 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -74,7 +74,7 @@ def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], Lis ) m_content = os.listdir(os.path.join(nfcore_modules_dir, m)) # Not a module, but contains sub-modules - if not "main.nf" in m_content: + if "main.nf" not in m_content: for tool in m_content: nfcore_modules_names.append(os.path.join(m, tool)) else: diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 5c50c53fb9..6394c8540c 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -9,8 +9,6 @@ from typing import Literal, Optional import questionary -import rich -import rich.columns import nf_core.list import nf_core.utils @@ -197,7 +195,6 @@ def format_param(self, name, properties, required_properties=(), show_hidden=Fal self.schema_obj.get_schema_defaults() default = properties.get("default") typ = properties.get("type") - required = name in required_properties out += _print_wrapped(name, "-", mode="both") diff --git a/nf_core/refgenie.py b/nf_core/refgenie.py index 6f09b75532..d4d9451f0e 100644 --- a/nf_core/refgenie.py +++ b/nf_core/refgenie.py @@ -2,7 +2,6 @@ Update a nextflow.config file with refgenie genomes """ -import json import logging import os import re diff --git a/nf_core/schema.py b/nf_core/schema.py index 7e4726f189..049842ed1f 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -317,7 +317,7 @@ def validate_config_default_parameter(self, param, schema_param, config_default) param ] = f"String should not be set to `{config_default}`" if schema_param["type"] == "boolean": - if not str(config_default) in ["false", "true"]: + if str(config_default) not in ["false", "true"]: self.invalid_nextflow_config_default_parameters[ param ] = f"Booleans should only be true or false, not `{config_default}`" diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 4944b26188..58f6afef90 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -118,7 +118,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): subworkflow.failed.append( ( "meta_modules_deprecated", - f"Deprecated section 'modules' found in `meta.yml`, use 'components' instead", + "Deprecated section 'modules' found in `meta.yml`, use 'components' instead", subworkflow.meta_yml, ) ) @@ -126,7 +126,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): subworkflow.passed.append( ( "meta_modules_deprecated", - f"Deprecated section 'modules' not found in `meta.yml`", + "Deprecated section 'modules' not found in `meta.yml`", subworkflow.meta_yml, ) ) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index a2107f633c..0a32c6c56b 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -6,7 +6,6 @@ from typing import Dict import git -import rich.progress from git.exc import GitCommandError from nf_core.utils import load_tools_config diff --git a/tests/modules/bump_versions.py b/tests/modules/bump_versions.py index 3c19041f63..e71c388c47 100644 --- a/tests/modules/bump_versions.py +++ b/tests/modules/bump_versions.py @@ -2,7 +2,6 @@ import re import pytest -import yaml import nf_core.modules from nf_core.modules.modules_utils import ModuleException diff --git a/tests/modules/create.py b/tests/modules/create.py index 74e5ec3896..29b027421a 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -1,4 +1,3 @@ -import filecmp import os import shutil from pathlib import Path diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index fc628df34f..88d889ef20 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -1,4 +1,3 @@ -import filecmp import os import shutil from pathlib import Path diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 1380db2260..94c27b60ab 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -1,4 +1,3 @@ -import os from pathlib import Path import pytest diff --git a/tests/subworkflows/remove.py b/tests/subworkflows/remove.py index 53a948778b..dec67875bd 100644 --- a/tests/subworkflows/remove.py +++ b/tests/subworkflows/remove.py @@ -1,7 +1,5 @@ from pathlib import Path -from rich.console import Console - from nf_core.modules.modules_json import ModulesJson @@ -18,7 +16,7 @@ def test_subworkflows_remove_subworkflow(self): bam_sort_stats_samtools_path = Path(subworkflow_path, "bam_sort_stats_samtools") bam_stats_samtools_path = Path(subworkflow_path, "bam_stats_samtools") samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index") - mod_json_obj = ModulesJson(self.pipeline_dir) + ModulesJson(self.pipeline_dir) mod_json_before = ModulesJson(self.pipeline_dir).get_modules_json() assert self.subworkflow_remove.remove("bam_sort_stats_samtools") mod_json_after = ModulesJson(self.pipeline_dir).get_modules_json() diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 698086e186..f9cc89bd43 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -8,7 +8,6 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE -from nf_core.modules.remove import ModuleRemove from nf_core.modules.update import ModuleUpdate from nf_core.subworkflows.update import SubworkflowUpdate diff --git a/tests/test_download.py b/tests/test_download.py index 7c9532e977..12ac453f88 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -1,7 +1,6 @@ """Tests for the download subcommand of nf-core tools """ -import hashlib import os import re import shutil @@ -16,9 +15,9 @@ import nf_core.utils from nf_core.download import ContainerError, DownloadWorkflow, WorkflowRepo from nf_core.synced_repo import SyncedRepo -from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, run_cmd +from nf_core.utils import run_cmd -from .utils import with_temporary_file, with_temporary_folder +from .utils import with_temporary_folder class DownloadTest(unittest.TestCase): @@ -343,7 +342,6 @@ def test_get_singularity_images(self, tmp_path, mock_fetch_wf_config): container_library=("mirage-the-imaginative-registry.io", "quay.io", "ghcr.io", "docker.io"), ) mock_fetch_wf_config.return_value = { - "process.mapping.container": "helloworld", "process.mapping.container": "helloworld", "process.mapping.container": "helloooooooworld", "process.mapping.container": "ewels/multiqc:gorewrite", diff --git a/tests/test_launch.py b/tests/test_launch.py index 03c6a8b692..4a07b255d9 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -86,7 +86,7 @@ def test_get_pipeline_defaults(self): self.launcher.get_pipeline_schema() self.launcher.set_schema_inputs() assert len(self.launcher.schema_obj.input_params) > 0 - assert self.launcher.schema_obj.input_params["validate_params"] == True + assert self.launcher.schema_obj.input_params["validate_params"] is True @with_temporary_file def test_get_pipeline_defaults_input_params(self, tmp_file): @@ -119,12 +119,12 @@ def test_ob_to_questionary_string(self): @mock.patch("questionary.unsafe_prompt", side_effect=[{"use_web_gui": "Web based"}]) def test_prompt_web_gui_true(self, mock_prompt): """Check the prompt to launch the web schema or use the cli""" - assert self.launcher.prompt_web_gui() == True + assert self.launcher.prompt_web_gui() is True @mock.patch("questionary.unsafe_prompt", side_effect=[{"use_web_gui": "Command line"}]) def test_prompt_web_gui_false(self, mock_prompt): """Check the prompt to launch the web schema or use the cli""" - assert self.launcher.prompt_web_gui() == False + assert self.launcher.prompt_web_gui() is False @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{}]) def test_launch_web_gui_missing_keys(self, mock_poll_nfcore_web_api): @@ -144,7 +144,7 @@ def test_launch_web_gui(self, mock_poll_nfcore_web_api, mock_webbrowser, mock_wa """Check the code that opens the web browser""" self.launcher.get_pipeline_schema() self.launcher.merge_nxf_flag_schema() - assert self.launcher.launch_web_gui() == None + assert self.launcher.launch_web_gui() is None @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "error", "message": "foo"}]) def test_get_web_launch_response_error(self, mock_poll_nfcore_web_api): @@ -163,7 +163,7 @@ def test_get_web_launch_response_unexpected(self, mock_poll_nfcore_web_api): @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "waiting_for_user"}]) def test_get_web_launch_response_waiting(self, mock_poll_nfcore_web_api): """Test polling the website for a launch response - status waiting_for_user""" - assert self.launcher.get_web_launch_response() == False + assert self.launcher.get_web_launch_response() is False @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "launch_params_complete"}]) def test_get_web_launch_response_missing_keys(self, mock_poll_nfcore_web_api): @@ -191,7 +191,7 @@ def test_get_web_launch_response_missing_keys(self, mock_poll_nfcore_web_api): def test_get_web_launch_response_valid(self, mock_poll_nfcore_web_api, mock_sanitise): """Test polling the website for a launch response - complete, valid response""" self.launcher.get_pipeline_schema() - assert self.launcher.get_web_launch_response() == True + assert self.launcher.get_web_launch_response() is True def test_sanitise_web_response(self): """Check that we can properly sanitise results from the web""" @@ -201,7 +201,7 @@ def test_sanitise_web_response(self): self.launcher.schema_obj.input_params["max_cpus"] = "12" self.launcher.sanitise_web_response() assert "-name" not in self.launcher.nxf_flags - assert self.launcher.schema_obj.input_params["igenomes_ignore"] == True + assert self.launcher.schema_obj.input_params["igenomes_ignore"] is True assert self.launcher.schema_obj.input_params["max_cpus"] == 12 def test_ob_to_questionary_bool(self): @@ -216,12 +216,12 @@ def test_ob_to_questionary_bool(self): assert result["message"] == "" assert result["choices"] == ["True", "False"] assert result["default"] == "True" - assert result["filter"]("True") == True - assert result["filter"]("true") == True - assert result["filter"](True) == True - assert result["filter"]("False") == False - assert result["filter"]("false") == False - assert result["filter"](False) == False + assert result["filter"]("True") is True + assert result["filter"]("true") is True + assert result["filter"](True) is True + assert result["filter"]("False") is False + assert result["filter"]("false") is False + assert result["filter"](False) is False def test_ob_to_questionary_number(self): """Check converting a python dict to a pyenquirer format - with enum""" diff --git a/tests/test_lint.py b/tests/test_lint.py index b2e7f3b574..3ed069ca8b 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -56,7 +56,7 @@ def test_run_linting_function(self): We don't really check any of this code as it's just a series of function calls and we're testing each of those individually. This is mostly to check for syntax errors.""" - lint_obj = nf_core.lint.run_linting(self.test_pipeline_dir, False) + nf_core.lint.run_linting(self.test_pipeline_dir, False) def test_init_PipelineLint(self): """Simply create a PipelineLint object. diff --git a/tests/test_params_file.py b/tests/test_params_file.py index 824e8fe345..c6aeab0e08 100644 --- a/tests/test_params_file.py +++ b/tests/test_params_file.py @@ -68,7 +68,7 @@ def test_build_template_file_exists(self, caplog): # Creates a new empty file outfile = Path(self.tmp_dir) / "params-file.yml" - with open(outfile, "w") as fp: + with open(outfile, "w"): pass res = self.params_template_builder.write_params_file(outfile) diff --git a/tests/test_refgenie.py b/tests/test_refgenie.py index 73fbcb863f..27aa4e3bc0 100644 --- a/tests/test_refgenie.py +++ b/tests/test_refgenie.py @@ -7,8 +7,6 @@ import tempfile import unittest -import yaml - class TestRefgenie(unittest.TestCase): """Class for refgenie tests""" diff --git a/tests/test_schema.py b/tests/test_schema.py index 105cd9473e..89fcc98b66 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -305,7 +305,7 @@ def test_build_schema(self): Build a new schema param from a pipeline Run code to ensure it doesn't crash. Individual functions tested separately. """ - param = self.schema_obj.build_schema(self.template_dir, True, False, None) + self.schema_obj.build_schema(self.template_dir, True, False, None) @with_temporary_folder def test_build_schema_from_scratch(self, tmp_dir): @@ -319,7 +319,7 @@ def test_build_schema_from_scratch(self, tmp_dir): shutil.copytree(self.template_dir, test_pipeline_dir) os.remove(os.path.join(test_pipeline_dir, "nextflow_schema.json")) - param = self.schema_obj.build_schema(test_pipeline_dir, True, False, None) + self.schema_obj.build_schema(test_pipeline_dir, True, False, None) @mock.patch("requests.post") def test_launch_web_builder_timeout(self, mock_post): diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index 154a31fca6..c7088b9282 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -1,8 +1,5 @@ -import tempfile from pathlib import Path -import pytest - from .utils import with_temporary_file, with_temporary_folder diff --git a/tests/utils.py b/tests/utils.py index 198ac3d583..44386759ed 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,9 +5,7 @@ import functools import os import tempfile -from contextlib import contextmanager -from pathlib import Path -from typing import Any, Callable, Generator, Tuple +from typing import Any, Callable, Tuple import responses From 8cc98a2ca27abd924d61d1f816bb29542ef1c345 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 15:24:25 +0100 Subject: [PATCH 09/22] manually fix other ruff errors --- docs/api/_src/conf.py | 3 +- nf_core/components/lint/__init__.py | 2 +- nf_core/components/patch.py | 4 +- nf_core/launch.py | 6 +- nf_core/lint/merge_markers.py | 14 ++--- nf_core/lint/pipeline_todos.py | 14 ++--- nf_core/lint/system_exit.py | 6 +- nf_core/lint/template_strings.py | 4 +- nf_core/list.py | 2 +- nf_core/modules/bump_versions.py | 8 +-- nf_core/modules/lint/main_nf.py | 94 ++++++++++++++-------------- nf_core/subworkflows/lint/main_nf.py | 36 +++++------ nf_core/sync.py | 4 +- nf_core/synced_repo.py | 3 +- nf_core/utils.py | 34 +++++----- tests/test_download.py | 8 +-- tests/test_refgenie.py | 2 +- tests/test_utils.py | 2 +- 18 files changed, 125 insertions(+), 121 deletions(-) diff --git a/docs/api/_src/conf.py b/docs/api/_src/conf.py index 27eaf9bcb3..a62b23cc06 100644 --- a/docs/api/_src/conf.py +++ b/docs/api/_src/conf.py @@ -16,9 +16,10 @@ import sys from typing import Dict -sys.path.insert(0, os.path.abspath("../../../nf_core")) import nf_core +sys.path.insert(0, os.path.abspath("../../../nf_core")) + # -- Project information ----------------------------------------------------- project = "nf-core/tools" diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index efffc28e85..d86b3ea5f5 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -216,7 +216,7 @@ def _print_results(self, show_passed=False, sort_by="test"): try: for lint_result in tests: max_name_len = max(len(lint_result.component_name), max_name_len) - except: + except Exception: pass # Helper function to format test links nicely diff --git a/nf_core/components/patch.py b/nf_core/components/patch.py index 28f2f886b1..55d5747451 100644 --- a/nf_core/components/patch.py +++ b/nf_core/components/patch.py @@ -35,7 +35,7 @@ def _parameter_checks(self, component): if component is not None and component not in component_names: component_dir = [dir for dir, m in components if m == component][0] raise UserWarning( - f"{self.component_type[:-1].title()} '{Path(self.component_type, component_dir, module)}' does not exist in the pipeline" + f"{self.component_type[:-1].title()} '{Path(self.component_type, component_dir, component)}' does not exist in the pipeline" ) def patch(self, component=None): @@ -220,5 +220,5 @@ def remove(self, component): ): log.error( f"Module files do not appear to match the remote for the commit sha in the 'module.json': {component_version}\n" - f"Recommend reinstalling with 'nf-core modules install --force --sha {component_version} {module}' " + f"Recommend reinstalling with 'nf-core modules install --force --sha {component_version} {component}' " ) diff --git a/nf_core/launch.py b/nf_core/launch.py index 363506c448..5fc358a51d 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -428,7 +428,7 @@ def prompt_param(self, param_id, param_obj, is_required, answers): answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) # If required and got an empty reponse, ask again - while type(answer[param_id]) is str and answer[param_id].strip() == "" and is_required: + while isinstance(answer[param_id], str) and answer[param_id].strip() == "" and is_required: log.error(f"'--{param_id}' is required") answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) @@ -546,14 +546,14 @@ def single_param_to_questionary(self, param_id, param_obj, answers=None, print_h # Start with the default from the param object if "default" in param_obj: # Boolean default is cast back to a string later - this just normalises all inputs - if param_obj["type"] == "boolean" and type(param_obj["default"]) is str: + if param_obj["type"] == "boolean" and isinstance(param_obj["default"], str): question["default"] = param_obj["default"].lower() == "true" else: question["default"] = param_obj["default"] # Overwrite default with parsed schema, includes --params-in etc if self.schema_obj is not None and param_id in self.schema_obj.input_params: - if param_obj["type"] == "boolean" and type(self.schema_obj.input_params[param_id]) is str: + if param_obj["type"] == "boolean" and isinstance(self.schema_obj.input_params[param_id], str): question["default"] = "true" == self.schema_obj.input_params[param_id].lower() else: question["default"] = self.schema_obj.input_params[param_id] diff --git a/nf_core/lint/merge_markers.py b/nf_core/lint/merge_markers.py index f33a5095d8..7b0eea69cf 100644 --- a/nf_core/lint/merge_markers.py +++ b/nf_core/lint/merge_markers.py @@ -24,8 +24,8 @@ def merge_markers(self): ignore = [".git"] if os.path.isfile(os.path.join(self.wf_path, ".gitignore")): with io.open(os.path.join(self.wf_path, ".gitignore"), "rt", encoding="latin1") as fh: - for l in fh: - ignore.append(os.path.basename(l.strip().rstrip("/"))) + for line in fh: + ignore.append(os.path.basename(line.strip().rstrip("/"))) for root, dirs, files in os.walk(self.wf_path, topdown=True): # Ignore files for i_base in ignore: @@ -42,11 +42,11 @@ def merge_markers(self): continue try: with io.open(os.path.join(root, fname), "rt", encoding="latin1") as fh: - for l in fh: - if ">>>>>>>" in l: - failed.append(f"Merge marker '>>>>>>>' in `{os.path.join(root, fname)}`: {l[:30]}") - if "<<<<<<<" in l: - failed.append(f"Merge marker '<<<<<<<' in `{os.path.join(root, fname)}`: {l[:30]}") + for line in fh: + if ">>>>>>>" in line: + failed.append(f"Merge marker '>>>>>>>' in `{os.path.join(root, fname)}`: {line[:30]}") + if "<<<<<<<" in line: + failed.append(f"Merge marker '<<<<<<<' in `{os.path.join(root, fname)}`: {line[:30]}") except FileNotFoundError: log.debug(f"Could not open file {os.path.join(root, fname)} in merge_markers lint test") if len(failed) == 0: diff --git a/nf_core/lint/pipeline_todos.py b/nf_core/lint/pipeline_todos.py index 890e227fa1..50afc0bc4a 100644 --- a/nf_core/lint/pipeline_todos.py +++ b/nf_core/lint/pipeline_todos.py @@ -42,8 +42,8 @@ def pipeline_todos(self, root_dir=None): ignore = [".git"] if os.path.isfile(os.path.join(root_dir, ".gitignore")): with io.open(os.path.join(root_dir, ".gitignore"), "rt", encoding="latin1") as fh: - for l in fh: - ignore.append(os.path.basename(l.strip().rstrip("/"))) + for line in fh: + ignore.append(os.path.basename(line.strip().rstrip("/"))) for root, dirs, files in os.walk(root_dir, topdown=True): # Ignore files for i_base in ignore: @@ -53,17 +53,17 @@ def pipeline_todos(self, root_dir=None): for fname in files: try: with io.open(os.path.join(root, fname), "rt", encoding="latin1") as fh: - for l in fh: - if "TODO nf-core" in l: - l = ( - l.replace("", "") .replace("# TODO nf-core: ", "") .replace("// TODO nf-core: ", "") .replace("TODO nf-core: ", "") .strip() ) - warned.append(f"TODO string in `{fname}`: _{l}_") + warned.append(f"TODO string in `{fname}`: _{line}_") file_paths.append(os.path.join(root, fname)) except FileNotFoundError: log.debug(f"Could not open file {fname} in pipeline_todos lint test") diff --git a/nf_core/lint/system_exit.py b/nf_core/lint/system_exit.py index 10c8d0bb4e..435a2452d0 100644 --- a/nf_core/lint/system_exit.py +++ b/nf_core/lint/system_exit.py @@ -25,9 +25,9 @@ def system_exit(self): for file in to_check: try: with file.open() as fh: - for i, l in enumerate(fh.readlines(), start=1): - if "System.exit" in l and "System.exit(0)" not in l: - warned.append(f"`System.exit` in {file.name}: _{l.strip()}_ [line {i}]") + for i, line in enumerate(fh.readlines(), start=1): + if "System.exit" in line and "System.exit(0)" not in line: + warned.append(f"`System.exit` in {file.name}: _{line.strip()}_ [line {i}]") except FileNotFoundError: log.debug(f"Could not open file {file.name} in system_exit lint test") diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py index fb1f0f32e5..4d827d24c4 100644 --- a/nf_core/lint/template_strings.py +++ b/nf_core/lint/template_strings.py @@ -32,9 +32,9 @@ def template_strings(self): with io.open(fn, "r", encoding="latin1") as fh: lnum = 0 - for l in fh: + for line in fh: lnum += 1 - cc_matches = re.findall(r"[^$]{{[^:}]*}}", l) + cc_matches = re.findall(r"[^$]{{[^:}]*}}", line) if len(cc_matches) > 0: for cc_match in cc_matches: failed.append(f"Found a Jinja template string in `{fn}` L{lnum}: {cc_match}") diff --git a/nf_core/list.py b/nf_core/list.py index 94d9d8e043..97f8e2d9a7 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -205,7 +205,7 @@ def print_summary(self): def sort_pulled_date(wf): try: return wf.local_wf.last_pull * -1 - except: + except Exception: return 0 filtered_workflows.sort(key=sort_pulled_date) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index b160466bb3..ea45deb332 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -146,9 +146,9 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: # try it in the main.nf instead try: with open(module.main_nf, "r") as fh: - for l in fh: - if "bioconda::" in l: - bioconda_packages = [b for b in l.split() if "bioconda::" in b] + for line in fh: + if "bioconda::" in line: + bioconda_packages = [b for b in line.split() if "bioconda::" in b] except FileNotFoundError: log.error( f"Neither `environment.yml` nor `main.nf` of {module.component_name} module could be read to get bioconada version of used tools." @@ -288,7 +288,7 @@ def _print_results(self) -> None: for m in [self.up_to_date, self.updated, self.failed]: try: max_mod_name_len = max(len(m[2]), max_mod_name_len) - except: + except Exception: pass def format_result(module_updates: List[Tuple[str, str]], table: Table) -> Table: diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 4d06589ced..d79eacb48e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -81,39 +81,39 @@ def main_nf(module_lint_object, module, fix_version, registry, progress_bar): script_lines = [] shell_lines = [] when_lines = [] - for l in lines: - if re.search(r"^\s*process\s*\w*\s*{", l) and state == "module": + for line in lines: + if re.search(r"^\s*process\s*\w*\s*{", line) and state == "module": state = "process" - if re.search(r"input\s*:", l) and state in ["process"]: + if re.search(r"input\s*:", line) and state in ["process"]: state = "input" continue - if re.search(r"output\s*:", l) and state in ["input", "process"]: + if re.search(r"output\s*:", line) and state in ["input", "process"]: state = "output" continue - if re.search(r"when\s*:", l) and state in ["input", "output", "process"]: + if re.search(r"when\s*:", line) and state in ["input", "output", "process"]: state = "when" continue - if re.search(r"script\s*:", l) and state in ["input", "output", "when", "process"]: + if re.search(r"script\s*:", line) and state in ["input", "output", "when", "process"]: state = "script" continue - if re.search(r"shell\s*:", l) and state in ["input", "output", "when", "process"]: + if re.search(r"shell\s*:", line) and state in ["input", "output", "when", "process"]: state = "shell" continue # Perform state-specific linting checks - if state == "process" and not _is_empty(l): - process_lines.append(l) - if state == "input" and not _is_empty(l): - inputs.extend(_parse_input(module, l)) - if state == "output" and not _is_empty(l): - outputs += _parse_output(module, l) + if state == "process" and not _is_empty(line): + process_lines.append(line) + if state == "input" and not _is_empty(line): + inputs.extend(_parse_input(module, line)) + if state == "output" and not _is_empty(line): + outputs += _parse_output(module, line) outputs = list(set(outputs)) # remove duplicate 'meta's - if state == "when" and not _is_empty(l): - when_lines.append(l) - if state == "script" and not _is_empty(l): - script_lines.append(l) - if state == "shell" and not _is_empty(l): - shell_lines.append(l) + if state == "when" and not _is_empty(line): + when_lines.append(line) + if state == "script" and not _is_empty(line): + script_lines.append(line) + if state == "shell" and not _is_empty(line): + shell_lines.append(line) # Check that we have required sections if not len(outputs): @@ -140,7 +140,7 @@ def main_nf(module_lint_object, module, fix_version, registry, progress_bar): # Check that shell uses a template if len(shell_lines): - if any("template" in l for l in shell_lines): + if any("template" in line for line in shell_lines): module.passed.append(("main_nf_shell_template", "`template` found in `shell` block", module.main_nf)) else: module.failed.append(("main_nf_shell_template", "No `template` found in `shell` block", module.main_nf)) @@ -249,16 +249,16 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): # Deprecated enable_conda for i, raw_line in enumerate(lines): url = None - l = raw_line.strip(" \n'\"}:") + line = raw_line.strip(" \n'\"}:") # Catch preceeding "container " - if l.startswith("container"): - l = l.replace("container", "").strip(" \n'\"}:") + if line.startswith("container"): + line = line.replace("container", "").strip(" \n'\"}:") - if _container_type(l) == "conda": - if "bioconda::" in l: - bioconda_packages = [b for b in l.split() if "bioconda::" in b] - match = re.search(r"params\.enable_conda", l) + if _container_type(line) == "conda": + if "bioconda::" in line: + bioconda_packages = [b for b in line.split() if "bioconda::" in b] + match = re.search(r"params\.enable_conda", line) if match is None: self.passed.append( ( @@ -275,31 +275,31 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): self.main_nf, ) ) - if _container_type(l) == "singularity": + if _container_type(line) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0 -> 0.11.9--0 # Please god let's find a better way to do this than regex - match = re.search(r"(?:[:.])?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l) + match = re.search(r"(?:[:.])?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", line) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) else: self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf)) singularity_tag = None - url = urlparse(l.split("'")[0]) + url = urlparse(line.split("'")[0]) - if _container_type(l) == "docker": + if _container_type(line) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5 -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1 -> v1.2.0_cv1 - match = re.search(r":([A-Za-z\d\-_.]+)$", l) + match = re.search(r":([A-Za-z\d\-_.]+)$", line) if match is not None: docker_tag = match.group(1) self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = None - if l.startswith(registry): - l_stripped = re.sub(r"\W+$", "", l) + if line.startswith(registry): + l_stripped = re.sub(r"\W+$", "", line) self.failed.append( ( "container_links", @@ -312,11 +312,11 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04) # If so, add quay.io as default container prefix - if l.count("/") == 1 and l.count(":") == 1: - l = "/".join([registry, l]).replace("//", "/") - url = urlparse(l.split("'")[0]) + if line.count("/") == 1 and line.count(":") == 1: + line = "/".join([registry, line]).replace("//", "/") + url = urlparse(line.split("'")[0]) - if l.startswith("container") or _container_type(l) == "docker" or _container_type(l) == "singularity": + if line.startswith("container") or _container_type(line) == "docker" or _container_type(line) == "singularity": check_container_link_line(self, raw_line, registry) # Try to connect to container URLs @@ -424,7 +424,7 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): def check_process_labels(self, lines): correct_process_labels = ["process_single", "process_low", "process_medium", "process_high", "process_long"] - all_labels = [l.strip() for l in lines if l.lstrip().startswith("label ")] + all_labels = [line.strip() for line in lines if line.lstrip().startswith("label ")] bad_labels = [] good_labels = [] if len(all_labels) > 0: @@ -475,14 +475,14 @@ def check_process_labels(self, lines): def check_container_link_line(self, raw_line, registry): """Look for common problems in the container name / URL, for docker and singularity.""" - l = raw_line.strip(" \n'\"}:") + line = raw_line.strip(" \n'\"}:") # lint double quotes - if l.count('"') > 2: + if line.count('"') > 2: self.failed.append( ( "container_links", - f"Too many double quotes found when specifying container: {l.lstrip('container ')}", + f"Too many double quotes found when specifying container: {line.lstrip('container ')}", self.main_nf, ) ) @@ -490,7 +490,7 @@ def check_container_link_line(self, raw_line, registry): self.passed.append( ( "container_links", - f"Correct number of double quotes found when specifying container: {l.lstrip('container ')}", + f"Correct number of double quotes found when specifying container: {line.lstrip('container ')}", self.main_nf, ) ) @@ -524,7 +524,9 @@ def check_container_link_line(self, raw_line, registry): ) # lint more than one container in the same line - if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or l.startswith(registry)): + if ("https://containers" in line or "https://depot" in line) and ( + "biocontainers/" in line or line.startswith(registry) + ): self.warned.append( ( "container_links", @@ -611,8 +613,8 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, # Check if the new version + build exist and replace new_lines = [] for line in lines: - l = line.strip(" '\"") - build_type = _container_type(l) + line_stripped = line.strip(" '\"") + build_type = _container_type(line_stripped) if build_type == "conda": new_lines.append(re.sub(rf"{current_version}", f"{latest_version}", line)) elif build_type in ("singularity", "docker"): diff --git a/nf_core/subworkflows/lint/main_nf.py b/nf_core/subworkflows/lint/main_nf.py index c7ce77490d..d730e57ad0 100644 --- a/nf_core/subworkflows/lint/main_nf.py +++ b/nf_core/subworkflows/lint/main_nf.py @@ -45,30 +45,30 @@ def main_nf(_, subworkflow): subworkflow_lines = [] workflow_lines = [] main_lines = [] - for l in lines: - if re.search(r"^\s*workflow\s*\w*\s*{", l) and state == "subworkflow": + for line in lines: + if re.search(r"^\s*workflow\s*\w*\s*{", line) and state == "subworkflow": state = "workflow" - if re.search(r"take\s*:", l) and state in ["workflow"]: + if re.search(r"take\s*:", line) and state in ["workflow"]: state = "take" continue - if re.search(r"main\s*:", l) and state in ["take", "workflow"]: + if re.search(r"main\s*:", line) and state in ["take", "workflow"]: state = "main" continue - if re.search(r"emit\s*:", l) and state in ["take", "main", "workflow"]: + if re.search(r"emit\s*:", line) and state in ["take", "main", "workflow"]: state = "emit" continue # Perform state-specific linting checks - if state == "subworkflow" and not _is_empty(l): - subworkflow_lines.append(l) - if state == "workflow" and not _is_empty(l): - workflow_lines.append(l) - if state == "take" and not _is_empty(l): - inputs.extend(_parse_input(subworkflow, l)) - if state == "emit" and not _is_empty(l): - outputs.extend(_parse_output(subworkflow, l)) - if state == "main" and not _is_empty(l): - main_lines.append(l) + if state == "subworkflow" and not _is_empty(line): + subworkflow_lines.append(line) + if state == "workflow" and not _is_empty(line): + workflow_lines.append(line) + if state == "take" and not _is_empty(line): + inputs.extend(_parse_input(subworkflow, line)) + if state == "emit" and not _is_empty(line): + outputs.extend(_parse_output(subworkflow, line)) + if state == "main" and not _is_empty(line): + main_lines.append(line) # Check that we have required sections if not len(outputs): @@ -177,9 +177,9 @@ def check_subworkflow_section(self, lines): ) includes = [] - for l in lines: - if l.strip().startswith("include"): - component_name = l.split("{")[1].split("}")[0].strip() + for line in lines: + if line.strip().startswith("include"): + component_name = line.split("{")[1].split("}")[0].strip() if " as " in component_name: component_name = component_name.split(" as ")[1].strip() includes.append(component_name) diff --git a/nf_core/sync.py b/nf_core/sync.py index 5402a6121d..19e1e5429d 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -395,7 +395,7 @@ def close_open_template_merge_prs(self): try: list_prs_json = json.loads(list_prs_request.content) list_prs_pp = json.dumps(list_prs_json, indent=4) - except: + except Exception: list_prs_json = list_prs_request.content list_prs_pp = list_prs_request.content @@ -438,7 +438,7 @@ def close_open_pr(self, pr): try: pr_request_json = json.loads(pr_request.content) pr_request_pp = json.dumps(pr_request_json, indent=4) - except: + except Exception: pr_request_json = pr_request.content pr_request_pp = pr_request.content diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 0a32c6c56b..c3abb880c1 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -8,6 +8,7 @@ import git from git.exc import GitCommandError +from nf_core.modules.modules_utils import repo_full_name_from_remote from nf_core.utils import load_tools_config log = logging.getLogger(__name__) @@ -116,7 +117,7 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url - self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) + self.fullname = repo_full_name_from_remote(self.remote_url) self.setup_local_repo(remote_url, branch, hide_progress) diff --git a/nf_core/utils.py b/nf_core/utils.py index bcc8faa3fd..aebe975e25 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -139,7 +139,7 @@ def __init__(self, wf_path): try: repo = git.Repo(self.wf_path) self.git_sha = repo.head.object.hexsha - except: + except Exception: log.debug(f"Could not find git hash for pipeline: {self.wf_path}") # Overwrite if we have the last commit from the PR - otherwise we get a merge commit hash @@ -274,8 +274,8 @@ def fetch_wf_config(wf_path, cache_config=True): result = run_cmd("nextflow", f"config -flat {wf_path}") if result is not None: nfconfig_raw, _ = result - for l in nfconfig_raw.splitlines(): - ul = l.decode("utf-8") + for line in nfconfig_raw.splitlines(): + ul = line.decode("utf-8") try: k, v = ul.split(" = ", 1) config[k] = v.strip("'\"") @@ -287,8 +287,8 @@ def fetch_wf_config(wf_path, cache_config=True): try: main_nf = os.path.join(wf_path, "main.nf") with open(main_nf, "r") as fh: - for l in fh: - match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=", l) + for line in fh: + match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=", line) if match: config[match.group(1)] = "null" except FileNotFoundError as e: @@ -666,18 +666,18 @@ def parse_anaconda_licence(anaconda_response, version=None): # Clean up / standardise licence names clean_licences = [] - for l in licences: - l = re.sub(r"GNU General Public License v\d \(([^\)]+)\)", r"\1", l) - l = re.sub(r"GNU GENERAL PUBLIC LICENSE", "GPL", l, flags=re.IGNORECASE) - l = l.replace("GPL-", "GPLv") - l = re.sub(r"GPL\s*([\d\.]+)", r"GPL v\1", l) # Add v prefix to GPL version if none found - l = re.sub(r"GPL\s*v(\d).0", r"GPL v\1", l) # Remove superflous .0 from GPL version - l = re.sub(r"GPL \(([^\)]+)\)", r"GPL \1", l) - l = re.sub(r"GPL\s*v", "GPL v", l) # Normalise whitespace to one space between GPL and v - l = re.sub(r"\s*(>=?)\s*(\d)", r" \1\2", l) # Normalise whitespace around >= GPL versions - l = l.replace("Clause", "clause") # BSD capitilisation - l = re.sub(r"-only$", "", l) # Remove superflous GPL "only" version suffixes - clean_licences.append(l) + for license in licences: + license = re.sub(r"GNU General Public License v\d \(([^\)]+)\)", r"\1", license) + license = re.sub(r"GNU GENERAL PUBLIC LICENSE", "GPL", license, flags=re.IGNORECASE) + license = license.replace("GPL-", "GPLv") + license = re.sub(r"GPL\s*([\d\.]+)", r"GPL v\1", license) # Add v prefix to GPL version if none found + license = re.sub(r"GPL\s*v(\d).0", r"GPL v\1", license) # Remove superflous .0 from GPL version + license = re.sub(r"GPL \(([^\)]+)\)", r"GPL \1", license) + license = re.sub(r"GPL\s*v", "GPL v", license) # Normalise whitespace to one space between GPL and v + license = re.sub(r"\s*(>=?)\s*(\d)", r" \1\2", license) # Normalise whitespace around >= GPL versions + license = license.replace("Clause", "clause") # BSD capitilisation + license = re.sub(r"-only$", "", license) # Remove superflous GPL "only" version suffixes + clean_licences.append(license) return clean_licences diff --git a/tests/test_download.py b/tests/test_download.py index 12ac453f88..4629779e49 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -159,8 +159,8 @@ def test__find_container_images_config_nextflow(self, tmp_path, mock_fetch_wf_co if result is not None: nfconfig_raw, _ = result config = {} - for l in nfconfig_raw.splitlines(): - ul = l.decode("utf-8") + for line in nfconfig_raw.splitlines(): + ul = line.decode("utf-8") try: k, v = ul.split(" = ", 1) config[k] = v.strip("'\"") @@ -342,8 +342,8 @@ def test_get_singularity_images(self, tmp_path, mock_fetch_wf_config): container_library=("mirage-the-imaginative-registry.io", "quay.io", "ghcr.io", "docker.io"), ) mock_fetch_wf_config.return_value = { - "process.mapping.container": "helloworld", - "process.mapping.container": "helloooooooworld", + "process.helloworld.container": "helloworld", + "process.hellooworld.container": "helloooooooworld", "process.mapping.container": "ewels/multiqc:gorewrite", } download_obj.find_container_images("workflow") diff --git a/tests/test_refgenie.py b/tests/test_refgenie.py index 27aa4e3bc0..5440c1c477 100644 --- a/tests/test_refgenie.py +++ b/tests/test_refgenie.py @@ -24,7 +24,7 @@ def setUp(self): # avoids adding includeConfig statement to config file outside the current tmpdir try: self.NXF_HOME_ORIGINAL = os.environ["NXF_HOME"] - except: + except Exception: self.NXF_HOME_ORIGINAL = None os.environ["NXF_HOME"] = self.NXF_HOME diff --git a/tests/test_utils.py b/tests/test_utils.py index 90d1886dbd..3079d75808 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -134,7 +134,7 @@ def test_request_cant_create_cache(self, mock_mkd, mock_exists): def test_pip_package_pass(self): result = nf_core.utils.pip_package("multiqc=1.10") - assert type(result) == dict + assert isinstance(result, dict) @mock.patch("requests.get") def test_pip_package_timeout(self, mock_get): From f2f3645598a0e9c4d946c8b3608af9915bf5c136 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 15:52:53 +0100 Subject: [PATCH 10/22] remove unused fullname from SyncedRepo and fix circular import --- nf_core/synced_repo.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index c3abb880c1..ac0f467e66 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -8,7 +8,6 @@ import git from git.exc import GitCommandError -from nf_core.modules.modules_utils import repo_full_name_from_remote from nf_core.utils import load_tools_config log = logging.getLogger(__name__) @@ -117,8 +116,6 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url - self.fullname = repo_full_name_from_remote(self.remote_url) - self.setup_local_repo(remote_url, branch, hide_progress) config_fn, repo_config = load_tools_config(self.local_repo_dir) From 3579de65b3789247787eb3cb434a1daa6c17d122 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 16:02:54 +0100 Subject: [PATCH 11/22] locate ruff cache dir at ~/.cache/ruff --- nf_core/pipeline-template/pyproject.toml | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index be5c608791..0c1705f363 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -5,6 +5,7 @@ line-length = 120 target-version = "py38" ignore = ["E501"] select = ["I", "E4", "E7", "E9", "F"] +cache-dir = "~/.cache/ruff" [tool.ruff.isort] known-first-party = ["nf_core"] diff --git a/pyproject.toml b/pyproject.toml index 7c3029e79c..23ba0e152c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ line-length = 120 target-version = "py38" ignore = ["E501"] select = ["I", "E4", "E7", "E9", "F"] +cache-dir = "~/.cache/ruff" [tool.ruff.isort] known-first-party = ["nf_core"] From 691e6547472fe166ec50b4e13b6188ea10ebf254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 22 Dec 2023 16:03:53 +0100 Subject: [PATCH 12/22] Apply suggestions from code review --- .gitpod.yml | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 724d7e7735..91c04d7f94 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -21,4 +21,4 @@ vscode: # - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code - - - charliermarsh.ruff # Code linter Ruff + - charliermarsh.ruff # Code linter Ruff diff --git a/CHANGELOG.md b/CHANGELOG.md index 37fb75845b..a67038995b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ ### General -- Add Ruff linter and formatter +- Add Ruff linter and formatter replacing Blac, isort and pyupgrade ([#2620](https://github.com/nf-core/tools/pull/2620)) # [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20] From aaa5166d2a7c242b5ef9e927df1b4a1b4eb28254 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 22 Dec 2023 16:10:36 +0100 Subject: [PATCH 13/22] use required to create params file --- nf_core/params_file.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 6394c8540c..e6ef83b783 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -195,6 +195,7 @@ def format_param(self, name, properties, required_properties=(), show_hidden=Fal self.schema_obj.get_schema_defaults() default = properties.get("default") typ = properties.get("type") + required = name in required_properties out += _print_wrapped(name, "-", mode="both") @@ -204,6 +205,9 @@ def format_param(self, name, properties, required_properties=(), show_hidden=Fal if typ: out += _print_wrapped(f"Type: {typ}", mode="none", indent=4) + if required: + out += _print_wrapped("Required", mode="none", indent=4) + out += _print_wrapped("\n", mode="end") out += f"# {name} = {json.dumps(default)}\n" From b430bbf979c8768df14ca4ed7bb3fc0bbb641500 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 2 Jan 2024 09:23:56 +0100 Subject: [PATCH 14/22] don't run prettier on things handled by ruff --- .prettierignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.prettierignore b/.prettierignore index 344cafca6e..a55074abfb 100644 --- a/.prettierignore +++ b/.prettierignore @@ -6,3 +6,7 @@ testing nf_core/module-template/meta.yml nf_core/module-template/tests/tags.yml nf_core/subworkflow-template/tests/tags.yml +# don't run on things handled by ruff +*.py +*.pyc + From 49714968266eee6576b787e4da680f07d826ca7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 4 Jan 2024 09:50:24 +0100 Subject: [PATCH 15/22] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .github/workflows/create-lint-wf.yml | 3 ++- CHANGELOG.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index bd3c65859c..786a9a7f79 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -83,7 +83,8 @@ jobs: - name: Run Ruff check run: ruff check create-lint-wf/nf-core-testpipeline - + - name: Run Ruff format + run: ruff format create-lint-wf/nf-core-testpipeline - name: Run ECLint check run: editorconfig-checker -exclude README.md $(find nf-core-testpipeline/.* -type f | grep -v '.git\|.py\|md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') working-directory: create-lint-wf diff --git a/CHANGELOG.md b/CHANGELOG.md index a67038995b..4aa4e9a65d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ ### General -- Add Ruff linter and formatter replacing Blac, isort and pyupgrade ([#2620](https://github.com/nf-core/tools/pull/2620)) +- Add Ruff linter and formatter replacing Black, isort and pyupgrade ([#2620](https://github.com/nf-core/tools/pull/2620)) # [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20] From 0c3c7791e7f9e5568ff332adad8fa9a07b96d4e4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 4 Jan 2024 10:09:09 +0100 Subject: [PATCH 16/22] select E1, UP and N, don't ignore E501 --- nf_core/pipeline-template/pyproject.toml | 3 +-- pyproject.toml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 0c1705f363..984c091091 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -3,8 +3,7 @@ [tool.ruff] line-length = 120 target-version = "py38" -ignore = ["E501"] -select = ["I", "E4", "E7", "E9", "F"] +select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] cache-dir = "~/.cache/ruff" [tool.ruff.isort] diff --git a/pyproject.toml b/pyproject.toml index 23ba0e152c..a5459fa795 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,7 @@ norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".githu [tool.ruff] line-length = 120 target-version = "py38" -ignore = ["E501"] -select = ["I", "E4", "E7", "E9", "F"] +select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] cache-dir = "~/.cache/ruff" [tool.ruff.isort] From 4fc75f9c12b0d062a5cd20a5d2bb080934bc0b46 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 4 Jan 2024 10:11:33 +0100 Subject: [PATCH 17/22] run ruff check --fix --- docs/api/_src/conf.py | 1 - nf_core/bump_version.py | 2 +- nf_core/components/components_command.py | 4 ++-- nf_core/components/components_utils.py | 2 +- nf_core/components/create.py | 7 +++---- nf_core/components/info.py | 4 ++-- nf_core/components/lint/__init__.py | 1 - nf_core/components/nfcore_component.py | 10 +++++----- nf_core/create.py | 4 ++-- nf_core/download.py | 9 ++++----- nf_core/launch.py | 1 - nf_core/licences.py | 1 - nf_core/lint/actions_awsfulltest.py | 2 +- nf_core/lint/actions_awstest.py | 2 +- nf_core/lint/actions_ci.py | 2 +- nf_core/lint/actions_schema_validation.py | 2 +- nf_core/lint/files_unchanged.py | 6 +++--- nf_core/lint/merge_markers.py | 5 ++--- nf_core/lint/multiqc_config.py | 2 +- nf_core/lint/nextflow_config.py | 4 ++-- nf_core/lint/pipeline_todos.py | 5 ++--- nf_core/lint/readme.py | 2 +- nf_core/lint/template_strings.py | 3 +-- nf_core/lint/version_consistency.py | 2 +- nf_core/list.py | 1 - nf_core/modules/bump_versions.py | 10 ++++------ nf_core/modules/lint/__init__.py | 1 - nf_core/modules/lint/environment_yml.py | 10 ++++------ nf_core/modules/lint/main_nf.py | 6 +++--- nf_core/modules/lint/meta_yml.py | 4 ++-- nf_core/modules/lint/module_patch.py | 6 +++--- nf_core/modules/lint/module_tests.py | 8 ++++---- nf_core/modules/modules_differ.py | 12 +++++------ nf_core/modules/modules_json.py | 2 +- nf_core/params_file.py | 1 - nf_core/refgenie.py | 2 +- nf_core/schema.py | 11 ++++------ nf_core/subworkflows/lint/__init__.py | 1 - nf_core/subworkflows/lint/main_nf.py | 2 +- nf_core/subworkflows/lint/meta_yml.py | 6 ++---- .../subworkflows/lint/subworkflow_tests.py | 8 ++++---- nf_core/sync.py | 2 +- nf_core/utils.py | 12 +++++------ tests/components/generate_snapshot.py | 8 ++++---- tests/lint/actions_awsfulltest.py | 4 ++-- tests/lint/actions_awstest.py | 2 +- tests/lint/actions_ci.py | 2 +- tests/lint/actions_schema_validation.py | 6 +++--- tests/lint/merge_markers.py | 2 +- tests/lint/multiqc_config.py | 8 ++++---- tests/lint/nextflow_config.py | 2 +- tests/modules/bump_versions.py | 4 ++-- tests/modules/create.py | 8 ++++---- tests/modules/lint.py | 10 +++++----- tests/modules/modules_json.py | 8 ++++---- tests/modules/patch.py | 20 +++++++++---------- tests/modules/update.py | 2 +- tests/subworkflows/create.py | 8 ++++---- tests/subworkflows/lint.py | 4 +--- tests/subworkflows/update.py | 2 +- tests/test_launch.py | 6 +++--- tests/test_lint.py | 2 +- tests/test_modules.py | 6 +++--- tests/test_params_file.py | 4 ++-- 64 files changed, 141 insertions(+), 165 deletions(-) diff --git a/docs/api/_src/conf.py b/docs/api/_src/conf.py index a62b23cc06..bfdbd7888d 100644 --- a/docs/api/_src/conf.py +++ b/docs/api/_src/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Configuration file for the Sphinx documentation builder. # diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index b8f19ffb65..c5e8931fbc 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -187,7 +187,7 @@ def update_file_version(filename: Union[str, Path], pipeline_obj: Pipeline, patt fn = pipeline_obj._fp(filename) content = "" try: - with open(fn, "r") as fh: + with open(fn) as fh: content = fh.read() except FileNotFoundError: log.warning(f"File not found: '{fn}'") diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 44924a2704..8332429835 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -227,7 +227,7 @@ def check_patch_paths(self, patch_path: Path, module_name: str) -> None: if patch_path.exists(): log.info(f"Modules {module_name} contains a patch file.") rewrite = False - with open(patch_path, "r") as fh: + with open(patch_path) as fh: lines = fh.readlines() for index, line in enumerate(lines): # Check if there are old paths in the patch file and replace @@ -264,7 +264,7 @@ def check_if_in_include_stmts(self, component_path: str) -> Dict[str, List[Dict[ if self.repo_type == "pipeline": workflow_files = Path(self.dir, "workflows").glob("*.nf") for workflow_file in workflow_files: - with open(workflow_file, "r") as fh: + with open(workflow_file) as fh: # Check if component path is in the file using mmap with mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ) as s: if s.find(component_path.encode()) != -1: diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 69f9452d5e..01650a643d 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -138,7 +138,7 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[str], List[str """ modules = [] subworkflows = [] - with open(Path(subworkflow_dir, "main.nf"), "r") as fh: + with open(Path(subworkflow_dir, "main.nf")) as fh: for line in fh: regex = re.compile( r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")" diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 568ca22af5..8ccd83b594 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -2,7 +2,6 @@ The ComponentCreate class handles generating of module and subworkflow templates """ -from __future__ import print_function import glob import json @@ -440,7 +439,7 @@ def _copy_old_files(self, component_old_path): pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) nextflow_config = pytest_dir / "nextflow.config" if nextflow_config.is_file(): - with open(nextflow_config, "r") as fh: + with open(nextflow_config) as fh: config_lines = "" for line in fh: if "publishDir" not in line: @@ -460,7 +459,7 @@ def _print_and_delete_pytest_files(self): "[violet]Do you want to delete the pytest files?[/]\nPytest file 'main.nf' will be printed to standard output to allow migrating the tests manually to 'main.nf.test'.", default=False, ): - with open(pytest_dir / "main.nf", "r") as fh: + with open(pytest_dir / "main.nf") as fh: log.info(fh.read()) shutil.rmtree(pytest_dir) log.info( @@ -475,7 +474,7 @@ def _print_and_delete_pytest_files(self): ) # Delete tags from pytest_modules.yml modules_yml = Path(self.directory, "tests", "config", "pytest_modules.yml") - with open(modules_yml, "r") as fh: + with open(modules_yml) as fh: yml_file = yaml.safe_load(fh) yml_key = str(self.component_dir) if self.component_type == "modules" else f"subworkflows/{self.component_dir}" if yml_key in yml_file: diff --git a/nf_core/components/info.py b/nf_core/components/info.py index e4d8038b87..54fc0004dc 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -184,7 +184,7 @@ def get_local_yaml(self): meta_fn = Path(comp_dir, "meta.yml") if meta_fn.exists(): log.debug(f"Found local file: {meta_fn}") - with open(meta_fn, "r") as fh: + with open(meta_fn) as fh: self.local_path = comp_dir return yaml.safe_load(fh) @@ -196,7 +196,7 @@ def get_local_yaml(self): meta_fn = Path(comp_dir, "meta.yml") if meta_fn.exists(): log.debug(f"Found local file: {meta_fn}") - with open(meta_fn, "r") as fh: + with open(meta_fn) as fh: self.local_path = comp_dir return yaml.safe_load(fh) log.debug(f"{self.component_type[:-1].title()} '{self.component}' meta.yml not found locally") diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index d86b3ea5f5..d4867a7067 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -3,7 +3,6 @@ in nf-core pipelines """ -from __future__ import print_function import logging import operator diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 874fa570bc..e7e12c7837 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -81,7 +81,7 @@ def __init__( def _get_main_nf_tags(self, test_main_nf: Union[Path, str]): """Collect all tags from the main.nf.test file.""" tags = [] - with open(test_main_nf, "r") as fh: + with open(test_main_nf) as fh: for line in fh: if line.strip().startswith("tag"): tags.append(line.strip().split()[1].strip('"')) @@ -90,7 +90,7 @@ def _get_main_nf_tags(self, test_main_nf: Union[Path, str]): def _get_included_components(self, main_nf: Union[Path, str]): """Collect all included components from the main.nf file.""" included_components = [] - with open(main_nf, "r") as fh: + with open(main_nf) as fh: for line in fh: if line.strip().startswith("include"): # get tool/subtool or subworkflow name from include statement, can be in the form @@ -107,7 +107,7 @@ def _get_included_components(self, main_nf: Union[Path, str]): def _get_included_components_in_chained_tests(self, main_nf_test: Union[Path, str]): """Collect all included components from the main.nf file.""" included_components = [] - with open(main_nf_test, "r") as fh: + with open(main_nf_test) as fh: for line in fh: if line.strip().startswith("script"): # get tool/subtool or subworkflow name from script statement, can be: @@ -151,7 +151,7 @@ def _get_included_components_in_chained_tests(self, main_nf_test: Union[Path, st def get_inputs_from_main_nf(self): """Collect all inputs from the main.nf file.""" inputs = [] - with open(self.main_nf, "r") as f: + with open(self.main_nf) as f: data = f.read() # get input values from main.nf after "input:", which can be formatted as tuple val(foo) path(bar) or val foo or val bar or path bar or path foo # regex matches: @@ -178,7 +178,7 @@ def get_inputs_from_main_nf(self): def get_outputs_from_main_nf(self): outputs = [] - with open(self.main_nf, "r") as f: + with open(self.main_nf) as f: data = f.read() # get output values from main.nf after "output:". the names are always after "emit:" if "output:" not in data: diff --git a/nf_core/create.py b/nf_core/create.py index 56d0912a07..0d5b7e4d7b 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -108,7 +108,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa # Obtain template customization info from template yaml file or `.nf-core.yml` config file try: if template_yaml_path is not None: - with open(template_yaml_path, "r") as f: + with open(template_yaml_path) as f: template_yaml = yaml.safe_load(f) elif "template" in config_yml: template_yaml = config_yml["template"] @@ -395,7 +395,7 @@ def remove_nf_core_in_bug_report_template(self): """ bug_report_path = self.outdir / ".github" / "ISSUE_TEMPLATE" / "bug_report.yml" - with open(bug_report_path, "r") as fh: + with open(bug_report_path) as fh: contents = yaml.load(fh, Loader=yaml.FullLoader) # Remove the first item in the body, which is the information about the docs diff --git a/nf_core/download.py b/nf_core/download.py index 1b05c266ec..86de160858 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1,6 +1,5 @@ """Downloads a nf-core pipeline to the local file system.""" -from __future__ import print_function import concurrent.futures import io @@ -640,7 +639,7 @@ def wf_use_local_configs(self, revision_dirname): log.debug(f"Editing 'params.custom_config_base' in '{nfconfig_fn}'") # Load the nextflow.config file into memory - with open(nfconfig_fn, "r") as nfconfig_fh: + with open(nfconfig_fn) as nfconfig_fh: nfconfig = nfconfig_fh.read() # Replace the target string @@ -700,7 +699,7 @@ def find_container_images(self, workflow_directory): if bool(config_findings_dsl2): # finding fill always be a tuple of length 2, first the quote used and second the enquoted value. for finding in config_findings_dsl2: - config_findings.append((finding + (self.nf_config, "Nextflow configs"))) + config_findings.append(finding + (self.nf_config, "Nextflow configs")) else: # no regex match, likely just plain string """ Append string also as finding-like tuple for consistency @@ -719,7 +718,7 @@ def find_container_images(self, workflow_directory): for file in files: if file.endswith(".nf"): file_path = os.path.join(subdir, file) - with open(file_path, "r") as fh: + with open(file_path) as fh: # Look for any lines with container "xxx" or container 'xxx' search_space = fh.read() """ @@ -744,7 +743,7 @@ def find_container_images(self, workflow_directory): for finding in local_module_findings: # append finding since we want to collect them from all modules # also append search_space because we need to start over later if nothing was found. - module_findings.append((finding + (search_space, file_path))) + module_findings.append(finding + (search_space, file_path)) # Not sure if there will ever be multiple container definitions per module, but beware DSL3. # Like above run on shallow copy, because length may change at runtime. diff --git a/nf_core/launch.py b/nf_core/launch.py index 5fc358a51d..25bb4c150c 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -1,6 +1,5 @@ """ Launch a pipeline, interactively collecting params """ -from __future__ import print_function import copy import json diff --git a/nf_core/licences.py b/nf_core/licences.py index d686a56178..a8a35334dd 100644 --- a/nf_core/licences.py +++ b/nf_core/licences.py @@ -1,6 +1,5 @@ """Lists software licences for a given workflow.""" -from __future__ import print_function import json import logging diff --git a/nf_core/lint/actions_awsfulltest.py b/nf_core/lint/actions_awsfulltest.py index e8e1c951b1..66aa3f99bf 100644 --- a/nf_core/lint/actions_awsfulltest.py +++ b/nf_core/lint/actions_awsfulltest.py @@ -32,7 +32,7 @@ def actions_awsfulltest(self): fn = os.path.join(self.wf_path, ".github", "workflows", "awsfulltest.yml") if os.path.isfile(fn): try: - with open(fn, "r") as fh: + with open(fn) as fh: wf = yaml.safe_load(fh) except Exception as e: return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} diff --git a/nf_core/lint/actions_awstest.py b/nf_core/lint/actions_awstest.py index ccdf0abf6a..7c55998944 100644 --- a/nf_core/lint/actions_awstest.py +++ b/nf_core/lint/actions_awstest.py @@ -27,7 +27,7 @@ def actions_awstest(self): return {"ignored": [f"'awstest.yml' workflow not found: `{fn}`"]} try: - with open(fn, "r") as fh: + with open(fn) as fh: wf = yaml.safe_load(fh) except Exception as e: return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} diff --git a/nf_core/lint/actions_ci.py b/nf_core/lint/actions_ci.py index 1c0a505fa2..a3e7d54b66 100644 --- a/nf_core/lint/actions_ci.py +++ b/nf_core/lint/actions_ci.py @@ -47,7 +47,7 @@ def actions_ci(self): return {"ignored": ["'.github/workflows/ci.yml' not found"]} try: - with open(fn, "r") as fh: + with open(fn) as fh: ciwf = yaml.safe_load(fh) except Exception as e: return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} diff --git a/nf_core/lint/actions_schema_validation.py b/nf_core/lint/actions_schema_validation.py index 9d49b84c6b..fa4471d98c 100644 --- a/nf_core/lint/actions_schema_validation.py +++ b/nf_core/lint/actions_schema_validation.py @@ -36,7 +36,7 @@ def actions_schema_validation(self): # load workflow try: - with open(wf_path, "r") as fh: + with open(wf_path) as fh: wf_json = yaml.safe_load(fh) except Exception as e: failed.append(f"Could not parse yaml file: {wf}, {e}") diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 2b64d62638..82b286fb44 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -187,16 +187,16 @@ def _tf(file_path): else: for f in files: try: - with open(_pf(f), "r") as fh: + with open(_pf(f)) as fh: pipeline_file = fh.read() - with open(_tf(f), "r") as fh: + with open(_tf(f)) as fh: template_file = fh.read() if template_file in pipeline_file: passed.append(f"`{f}` matches the template") else: if "files_unchanged" in self.fix: # Try to fix the problem by overwriting the pipeline file - with open(_tf(f), "r") as fh: + with open(_tf(f)) as fh: template_file = fh.read() with open(_pf(f), "w") as fh: fh.write(template_file) diff --git a/nf_core/lint/merge_markers.py b/nf_core/lint/merge_markers.py index 7b0eea69cf..8ef425234b 100644 --- a/nf_core/lint/merge_markers.py +++ b/nf_core/lint/merge_markers.py @@ -1,5 +1,4 @@ import fnmatch -import io import logging import os @@ -23,7 +22,7 @@ def merge_markers(self): ignore = [".git"] if os.path.isfile(os.path.join(self.wf_path, ".gitignore")): - with io.open(os.path.join(self.wf_path, ".gitignore"), "rt", encoding="latin1") as fh: + with open(os.path.join(self.wf_path, ".gitignore"), encoding="latin1") as fh: for line in fh: ignore.append(os.path.basename(line.strip().rstrip("/"))) for root, dirs, files in os.walk(self.wf_path, topdown=True): @@ -41,7 +40,7 @@ def merge_markers(self): if nf_core.utils.is_file_binary(os.path.join(root, fname)): continue try: - with io.open(os.path.join(root, fname), "rt", encoding="latin1") as fh: + with open(os.path.join(root, fname), encoding="latin1") as fh: for line in fh: if ">>>>>>>" in line: failed.append(f"Merge marker '>>>>>>>' in `{os.path.join(root, fname)}`: {line[:30]}") diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index cbbeae07a8..b2f1a89a1b 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -34,7 +34,7 @@ def multiqc_config(self) -> Dict[str, List[str]]: return {"ignored": ["'assets/multiqc_config.yml' not found"]} try: - with open(fn, "r") as fh: + with open(fn) as fh: mqc_yml = yaml.safe_load(fh) except Exception as e: return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 24f1e5c12f..328bc03759 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -300,7 +300,7 @@ def nextflow_config(self): ] path = os.path.join(self.wf_path, "nextflow.config") i = 0 - with open(path, "r") as f: + with open(path) as f: for line in f: if lines[i] in line: i += 1 @@ -320,7 +320,7 @@ def nextflow_config(self): ) # Check for the availability of the "test" configuration profile by parsing nextflow.config - with open(os.path.join(self.wf_path, "nextflow.config"), "r") as f: + with open(os.path.join(self.wf_path, "nextflow.config")) as f: content = f.read() # Remove comments diff --git a/nf_core/lint/pipeline_todos.py b/nf_core/lint/pipeline_todos.py index 50afc0bc4a..ba6ec79150 100644 --- a/nf_core/lint/pipeline_todos.py +++ b/nf_core/lint/pipeline_todos.py @@ -1,5 +1,4 @@ import fnmatch -import io import logging import os @@ -41,7 +40,7 @@ def pipeline_todos(self, root_dir=None): ignore = [".git"] if os.path.isfile(os.path.join(root_dir, ".gitignore")): - with io.open(os.path.join(root_dir, ".gitignore"), "rt", encoding="latin1") as fh: + with open(os.path.join(root_dir, ".gitignore"), encoding="latin1") as fh: for line in fh: ignore.append(os.path.basename(line.strip().rstrip("/"))) for root, dirs, files in os.walk(root_dir, topdown=True): @@ -52,7 +51,7 @@ def pipeline_todos(self, root_dir=None): files[:] = [f for f in files if not fnmatch.fnmatch(os.path.join(root, f), i)] for fname in files: try: - with io.open(os.path.join(root, fname), "rt", encoding="latin1") as fh: + with open(os.path.join(root, fname), encoding="latin1") as fh: for line in fh: if "TODO nf-core" in line: line = ( diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py index 55060442b1..cade9ca3ea 100644 --- a/nf_core/lint/readme.py +++ b/nf_core/lint/readme.py @@ -31,7 +31,7 @@ def readme(self): # Remove field that should be ignored according to the linting config ignore_configs = self.lint_config.get("readme", []) - with open(os.path.join(self.wf_path, "README.md"), "r") as fh: + with open(os.path.join(self.wf_path, "README.md")) as fh: content = fh.read() if "nextflow_badge" not in ignore_configs: diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py index 4d827d24c4..3467229362 100644 --- a/nf_core/lint/template_strings.py +++ b/nf_core/lint/template_strings.py @@ -1,4 +1,3 @@ -import io import mimetypes import re @@ -30,7 +29,7 @@ def template_strings(self): if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): continue - with io.open(fn, "r", encoding="latin1") as fh: + with open(fn, encoding="latin1") as fh: lnum = 0 for line in fh: lnum += 1 diff --git a/nf_core/lint/version_consistency.py b/nf_core/lint/version_consistency.py index 4037bc01e4..e396ca9e7a 100644 --- a/nf_core/lint/version_consistency.py +++ b/nf_core/lint/version_consistency.py @@ -54,7 +54,7 @@ def version_consistency(self): if len(set(versions.values())) != 1: failed.append( "The versioning is not consistent between container, release tag " "and config. Found {}".format( - ", ".join(["{} = {}".format(k, v) for k, v in versions.items()]) + ", ".join([f"{k} = {v}" for k, v in versions.items()]) ) ) diff --git a/nf_core/list.py b/nf_core/list.py index 97f8e2d9a7..d0b59319a3 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -1,6 +1,5 @@ """Lists available nf-core pipelines and versions.""" -from __future__ import print_function import json import logging diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index ea45deb332..75ec15c206 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -4,8 +4,6 @@ """ -from __future__ import print_function - import logging import os import re @@ -145,7 +143,7 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: except FileNotFoundError: # try it in the main.nf instead try: - with open(module.main_nf, "r") as fh: + with open(module.main_nf) as fh: for line in fh: if "bioconda::" in line: bioconda_packages = [b for b in line.split() if "bioconda::" in b] @@ -207,7 +205,7 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: ), ] - with open(module.main_nf, "r") as fh: + with open(module.main_nf) as fh: content = fh.read() # Go over file content of main.nf and find replacements @@ -240,7 +238,7 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: fh.write(content) # change version in environment.yml - with open(module.environment_yml, "r") as fh: + with open(module.environment_yml) as fh: env_yml = yaml.safe_load(fh) re.sub(bioconda_packages[0], f"'bioconda::{bioconda_tool_name}={last_ver}'", env_yml["dependencies"]) with open(module.environment_yml, "w") as fh: @@ -265,7 +263,7 @@ def get_bioconda_version(self, module: NFCoreComponent) -> List[str]: # Check whether file exists and load it bioconda_packages = [] try: - with open(module.environment_yml, "r") as fh: + with open(module.environment_yml) as fh: env_yml = yaml.safe_load(fh) bioconda_packages = env_yml.get("dependencies", []) except FileNotFoundError: diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 68a38cc0cd..75d2d830b8 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -6,7 +6,6 @@ nf-core modules lint """ -from __future__ import print_function import logging import os diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index a052425539..c2fc9384e5 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -23,14 +23,14 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) env_yml = None # load the environment.yml file try: - with open(Path(module.component_dir, "environment.yml"), "r") as fh: + with open(Path(module.component_dir, "environment.yml")) as fh: env_yml = yaml.safe_load(fh) module.passed.append(("environment_yml_exists", "Module's `environment.yml` exists", module.environment_yml)) except FileNotFoundError: # check if the module's main.nf requires a conda environment - with open(Path(module.component_dir, "main.nf"), "r") as fh: + with open(Path(module.component_dir, "main.nf")) as fh: main_nf = fh.read() if 'conda "${moduleDir}/environment.yml"' in main_nf: module.failed.append( @@ -49,9 +49,7 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) if env_yml: valid_env_yml = False try: - with open( - Path(module_lint_object.modules_repo.local_repo_dir, "modules/environment-schema.json"), "r" - ) as fh: + with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/environment-schema.json")) as fh: schema = json.load(fh) validators.validate(instance=env_yml, schema=schema) module.passed.append( @@ -92,7 +90,7 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) yaml.dump(env_yml, fh, Dumper=custom_yaml_dumper()) # Check that the name in the environment.yml file matches the name in the meta.yml file - with open(Path(module.component_dir, "meta.yml"), "r") as fh: + with open(Path(module.component_dir, "meta.yml")) as fh: meta_yml = yaml.safe_load(fh) if env_yml["name"] == meta_yml["name"]: diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index d79eacb48e..fd4d81f7f2 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -55,7 +55,7 @@ def main_nf(module_lint_object, module, fix_version, registry, progress_bar): if lines is None: try: # Check whether file exists and load it - with open(module.main_nf, "r") as fh: + with open(module.main_nf) as fh: lines = fh.readlines() module.passed.append(("main_nf_exists", "Module file exists", module.main_nf)) except FileNotFoundError: @@ -348,7 +348,7 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): # Get bioconda packages from environment.yml try: - with open(Path(self.component_dir, "environment.yml"), "r") as fh: + with open(Path(self.component_dir, "environment.yml")) as fh: env_yml = yaml.safe_load(fh) if "dependencies" in env_yml: bioconda_packages = [x for x in env_yml["dependencies"] if isinstance(x, str) and "bioconda::" in x] @@ -607,7 +607,7 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, # Get latest build build = _get_build(response) - with open(self.main_nf, "r") as source: + with open(self.main_nf) as source: lines = source.readlines() # Check if the new version + build exist and replace diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 7552c1ceae..551a978f4d 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -54,7 +54,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None meta_yaml = yaml.safe_load("".join(lines)) if meta_yaml is None: try: - with open(module.meta_yml, "r") as fh: + with open(module.meta_yml) as fh: meta_yaml = yaml.safe_load(fh) module.passed.append(("meta_yml_exists", "Module `meta.yml` exists", module.meta_yml)) except FileNotFoundError: @@ -64,7 +64,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None # Confirm that the meta.yml file is valid according to the JSON schema valid_meta_yml = False try: - with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/meta-schema.json"), "r") as fh: + with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/meta-schema.json")) as fh: schema = json.load(fh) validators.validate(instance=meta_yaml, schema=schema) module.passed.append(("meta_yml_valid", "Module `meta.yml` is valid", module.meta_yml)) diff --git a/nf_core/modules/lint/module_patch.py b/nf_core/modules/lint/module_patch.py index d52962eabb..29bf78a66b 100644 --- a/nf_core/modules/lint/module_patch.py +++ b/nf_core/modules/lint/module_patch.py @@ -40,7 +40,7 @@ def check_patch_valid(module, patch_path): Returns: (bool): False if any test failed, True otherwise """ - with open(patch_path, "r") as fh: + with open(patch_path) as fh: patch_lines = fh.readlines() # Check that the file contains a patch for at least one file @@ -170,8 +170,8 @@ def patch_reversible(module_lint_object, module, patch_path): ) except LookupError: # Patch failed. Save the patch file by moving to the install dir - module.failed.append((("patch_reversible", "Patch file is outdated or edited", patch_path))) + module.failed.append(("patch_reversible", "Patch file is outdated or edited", patch_path)) return False - module.passed.append((("patch_reversible", "Patch agrees with module files", patch_path))) + module.passed.append(("patch_reversible", "Patch agrees with module files", patch_path)) return True diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 87033e3f49..520f8cf0a2 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -44,7 +44,7 @@ def module_tests(_, module: NFCoreComponent): if module.nftest_main_nf.is_file(): # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test - with open(module.nftest_main_nf, "r") as fh: + with open(module.nftest_main_nf) as fh: if "snapshot(" in fh.read(): snap_file = module.nftest_testdir / "main.nf.test.snap" if snap_file.is_file(): @@ -52,7 +52,7 @@ def module_tests(_, module: NFCoreComponent): ("test_snapshot_exists", "snapshot file `main.nf.test.snap` exists", snap_file) ) # Validate no empty files - with open(snap_file, "r") as snap_fh: + with open(snap_file) as snap_fh: try: snap_content = json.load(snap_fh) for test_name in snap_content.keys(): @@ -145,7 +145,7 @@ def module_tests(_, module: NFCoreComponent): pytest_yml_path = module.base_dir / "tests" / "config" / "pytest_modules.yml" if pytest_yml_path.is_file() and not is_pytest: try: - with open(pytest_yml_path, "r") as fh: + with open(pytest_yml_path) as fh: pytest_yml = yaml.safe_load(fh) if module.component_name in pytest_yml.keys(): module.failed.append( @@ -165,7 +165,7 @@ def module_tests(_, module: NFCoreComponent): if module.tags_yml.is_file(): # Check that tags.yml exists and it has the correct entry module.passed.append(("test_tags_yml_exists", "file `tags.yml` exists", module.tags_yml)) - with open(module.tags_yml, "r") as fh: + with open(module.tags_yml) as fh: tags_yml = yaml.safe_load(fh) if module.component_name in tags_yml.keys(): module.passed.append(("test_tags_yml", "correct entry in tags.yml", module.tags_yml)) diff --git a/nf_core/modules/modules_differ.py b/nf_core/modules/modules_differ.py index efce3868e5..a97229ff62 100644 --- a/nf_core/modules/modules_differ.py +++ b/nf_core/modules/modules_differ.py @@ -74,9 +74,9 @@ def get_module_diffs(from_dir, to_dir, for_git=True, dsp_from_dir=None, dsp_to_d temp_path = Path(to_dir, file) curr_path = Path(from_dir, file) if temp_path.exists() and curr_path.exists() and temp_path.is_file(): - with open(temp_path, "r") as fh: + with open(temp_path) as fh: new_lines = fh.readlines() - with open(curr_path, "r") as fh: + with open(curr_path) as fh: old_lines = fh.readlines() if new_lines == old_lines: @@ -93,7 +93,7 @@ def get_module_diffs(from_dir, to_dir, for_git=True, dsp_from_dir=None, dsp_to_d diffs[file] = (ModulesDiffer.DiffEnum.CHANGED, diff) elif temp_path.exists(): - with open(temp_path, "r") as fh: + with open(temp_path) as fh: new_lines = fh.readlines() # The file was created # Show file against /dev/null @@ -108,7 +108,7 @@ def get_module_diffs(from_dir, to_dir, for_git=True, dsp_from_dir=None, dsp_to_d elif curr_path.exists(): # The file was removed # Show file against /dev/null - with open(curr_path, "r") as fh: + with open(curr_path) as fh: old_lines = fh.readlines() diff = difflib.unified_diff( old_lines, @@ -279,7 +279,7 @@ def per_file_patch(patch_fn): dict[str, str]: A dictionary indexed by the filenames with the file patches as values """ - with open(patch_fn, "r") as fh: + with open(patch_fn) as fh: lines = fh.readlines() patches = {} @@ -447,7 +447,7 @@ def try_apply_patch(module, repo_path, patch_path, module_dir, reverse=False): log.debug(f"Applying patch to {file}") fn = Path(file).relative_to(module_relpath) file_path = module_dir / fn - with open(file_path, "r") as fh: + with open(file_path) as fh: file_lines = fh.readlines() patched_new_lines = ModulesDiffer.try_apply_single_patch(file_lines, patch, reverse=reverse) new_files[str(fn)] = patched_new_lines diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 32eb8736d6..f68c27b2d8 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -637,7 +637,7 @@ def load(self): UserWarning: If the modules.json file is not found """ try: - with open(self.modules_json_path, "r") as fh: + with open(self.modules_json_path) as fh: try: self.modules_json = json.load(fh) except json.JSONDecodeError as e: diff --git a/nf_core/params_file.py b/nf_core/params_file.py index e6ef83b783..bfb4ff7e85 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -1,6 +1,5 @@ """ Create a YAML parameter file """ -from __future__ import print_function import json import logging diff --git a/nf_core/refgenie.py b/nf_core/refgenie.py index d4d9451f0e..de9201bcd6 100644 --- a/nf_core/refgenie.py +++ b/nf_core/refgenie.py @@ -85,7 +85,7 @@ def _update_nextflow_home_config(refgenie_genomes_config_file, nxf_home): if os.path.exists(nxf_home_config): # look for include statement in config has_include_statement = False - with open(nxf_home_config, "r") as fh: + with open(nxf_home_config) as fh: lines = fh.readlines() for line in lines: if re.match(rf"\s*includeConfig\s*'{os.path.abspath(refgenie_genomes_config_file)}'", line): diff --git a/nf_core/schema.py b/nf_core/schema.py index 049842ed1f..b0c5dc04b6 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -1,6 +1,5 @@ """ Code to deal with pipeline JSON Schema """ -from __future__ import print_function import copy import json @@ -107,7 +106,7 @@ def load_lint_schema(self): def load_schema(self): """Load a pipeline schema from a file""" - with open(self.schema_filename, "r") as fh: + with open(self.schema_filename) as fh: self.schema = json.load(fh) self.schema_defaults = {} self.schema_params = {} @@ -189,7 +188,7 @@ def load_input_params(self, params_path): """ # First, try to load as JSON try: - with open(params_path, "r") as fh: + with open(params_path) as fh: try: params = json.load(fh) except json.JSONDecodeError as e: @@ -200,7 +199,7 @@ def load_input_params(self, params_path): log.debug(f"Could not load input params as JSON: {json_e}") # This failed, try to load as YAML try: - with open(params_path, "r") as fh: + with open(params_path) as fh: params = yaml.safe_load(fh) self.input_params.update(params) log.debug(f"Loaded YAML input params: {params_path}") @@ -757,9 +756,7 @@ def prompt_remove_schema_notfound_config(self, p_key): if self.no_prompts or self.schema_from_scratch: return True if Confirm.ask( - ":question: Unrecognised [bold]'params.{}'[/] found in the schema but not in the pipeline config! [yellow]Remove it?".format( - p_key - ) + f":question: Unrecognised [bold]'params.{p_key}'[/] found in the schema but not in the pipeline config! [yellow]Remove it?" ): return True return False diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index ffba41f9da..dc8fa68c30 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -6,7 +6,6 @@ nf-core subworkflows lint """ -from __future__ import print_function import logging import os diff --git a/nf_core/subworkflows/lint/main_nf.py b/nf_core/subworkflows/lint/main_nf.py index d730e57ad0..f59e1e4279 100644 --- a/nf_core/subworkflows/lint/main_nf.py +++ b/nf_core/subworkflows/lint/main_nf.py @@ -32,7 +32,7 @@ def main_nf(_, subworkflow): if lines is None: try: # Check whether file exists and load it - with open(subworkflow.main_nf, "r") as fh: + with open(subworkflow.main_nf) as fh: lines = fh.readlines() subworkflow.passed.append(("main_nf_exists", "Subworkflow file exists", subworkflow.main_nf)) except FileNotFoundError: diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 58f6afef90..24e75eddbf 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -26,7 +26,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): """ # Read the meta.yml file try: - with open(subworkflow.meta_yml, "r") as fh: + with open(subworkflow.meta_yml) as fh: meta_yaml = yaml.safe_load(fh) subworkflow.passed.append(("meta_yml_exists", "Subworkflow `meta.yml` exists", subworkflow.meta_yml)) except FileNotFoundError: @@ -36,9 +36,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): # Confirm that the meta.yml file is valid according to the JSON schema valid_meta_yml = True try: - with open( - Path(subworkflow_lint_object.modules_repo.local_repo_dir, "subworkflows/yaml-schema.json"), "r" - ) as fh: + with open(Path(subworkflow_lint_object.modules_repo.local_repo_dir, "subworkflows/yaml-schema.json")) as fh: schema = json.load(fh) jsonschema.validators.validate(instance=meta_yaml, schema=schema) subworkflow.passed.append(("meta_yml_valid", "Subworkflow `meta.yml` is valid", subworkflow.meta_yml)) diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 1ebced6d42..f7284320ea 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -52,14 +52,14 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): ) if subworkflow.nftest_main_nf.is_file(): - with open(subworkflow.nftest_main_nf, "r") as fh: + with open(subworkflow.nftest_main_nf) as fh: # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test if "snapshot(" in fh.read(): snap_file = subworkflow.nftest_testdir / "main.nf.test.snap" if snap_file.is_file(): subworkflow.passed.append(("test_snapshot_exists", "test `main.nf.test.snap` exists", snap_file)) # Validate no empty files - with open(snap_file, "r") as snap_fh: + with open(snap_file) as snap_fh: try: snap_content = json.load(snap_fh) for test_name in snap_content.keys(): @@ -158,7 +158,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): pytest_yml_path = subworkflow.base_dir / "tests" / "config" / "pytest_modules.yml" if pytest_yml_path.is_file() and not is_pytest: try: - with open(pytest_yml_path, "r") as fh: + with open(pytest_yml_path) as fh: pytest_yml = yaml.safe_load(fh) if "subworkflows/" + subworkflow.component_name in pytest_yml.keys(): subworkflow.failed.append( @@ -178,7 +178,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): if subworkflow.tags_yml.is_file(): # Check tags.yml exists and it has the correct entry subworkflow.passed.append(("test_tags_yml_exists", "file `tags.yml` exists", subworkflow.tags_yml)) - with open(subworkflow.tags_yml, "r") as fh: + with open(subworkflow.tags_yml) as fh: tags_yml = yaml.safe_load(fh) if "subworkflows/" + subworkflow.component_name in tags_yml.keys(): subworkflow.passed.append(("test_tags_yml", "correct entry in tags.yml", subworkflow.tags_yml)) diff --git a/nf_core/sync.py b/nf_core/sync.py index 19e1e5429d..63614c98c9 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -96,7 +96,7 @@ def __init__( default=False, ).unsafe_ask() if overwrite_template or "template" not in self.config_yml: - with open(template_yaml_path, "r") as f: + with open(template_yaml_path) as f: self.config_yml["template"] = yaml.safe_load(f) with open(self.config_yml_path, "w") as fh: yaml.safe_dump(self.config_yml, fh) diff --git a/nf_core/utils.py b/nf_core/utils.py index aebe975e25..121c2de0b5 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -188,7 +188,7 @@ def _load_pipeline_config(self): def _load_conda_environment(self): """Try to load the pipeline environment.yml file, if it exists""" try: - with open(os.path.join(self.wf_path, "environment.yml"), "r") as fh: + with open(os.path.join(self.wf_path, "environment.yml")) as fh: self.conda_config = yaml.safe_load(fh) except FileNotFoundError: log.debug("No conda `environment.yml` file found.") @@ -262,7 +262,7 @@ def fetch_wf_config(wf_path, cache_config=True): cache_path = os.path.join(cache_basedir, cache_fn) if os.path.isfile(cache_path) and cache_config is True: log.debug(f"Found a config cache, loading: {cache_path}") - with open(cache_path, "r") as fh: + with open(cache_path) as fh: try: config = json.load(fh) except json.JSONDecodeError as e: @@ -286,7 +286,7 @@ def fetch_wf_config(wf_path, cache_config=True): # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. try: main_nf = os.path.join(wf_path, "main.nf") - with open(main_nf, "r") as fh: + with open(main_nf) as fh: for line in fh: match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=", line) if match: @@ -480,7 +480,7 @@ def __call__(self, r): gh_cli_config_fn = os.path.expanduser("~/.config/gh/hosts.yml") if self.auth is None and os.path.exists(gh_cli_config_fn): try: - with open(gh_cli_config_fn, "r") as fh: + with open(gh_cli_config_fn) as fh: gh_cli_config = yaml.safe_load(fh) self.auth = requests.auth.HTTPBasicAuth( gh_cli_config["github.com"]["user"], gh_cli_config["github.com"]["oauth_token"] @@ -1025,7 +1025,7 @@ def load_tools_config(directory: Union[str, Path] = "."): log.debug(f"No tools config file found: {CONFIG_PATHS[0]}") return Path(directory, CONFIG_PATHS[0]), {} - with open(config_fn, "r") as fh: + with open(config_fn) as fh: tools_config = yaml.safe_load(fh) # If the file is empty @@ -1145,7 +1145,7 @@ def validate_file_md5(file_name, expected_md5hex): if file_md5hex.upper() == expected_md5hex.upper(): log.debug(f"md5 sum of image matches expected: {expected_md5hex}") else: - raise IOError(f"{file_name} md5 does not match remote: {expected_md5hex} - {file_md5hex}") + raise OSError(f"{file_name} md5 does not match remote: {expected_md5hex} - {file_md5hex}") return True diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index 46fd63fe3f..c5067d7210 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -26,7 +26,7 @@ def test_generate_snapshot_module(self): snap_path = Path("modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap") assert snap_path.exists() - with open(snap_path, "r") as fh: + with open(snap_path) as fh: snap_content = json.load(fh) assert "versions" in snap_content assert "content" in snap_content["versions"] @@ -48,7 +48,7 @@ def test_generate_snapshot_subworkflow(self): snap_path = Path("subworkflows", "nf-core-test", "bam_sort_stats_samtools", "tests", "main.nf.test.snap") assert snap_path.exists() - with open(snap_path, "r") as fh: + with open(snap_path) as fh: snap_content = json.load(fh) assert "test_bam_sort_stats_samtools_paired_end_flagstats" in snap_content assert ( @@ -86,7 +86,7 @@ def test_update_snapshot_module(self): with set_wd(self.nfcore_modules): snap_path = Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap") - with open(snap_path, "r") as fh: + with open(snap_path) as fh: snap_content = json.load(fh) original_timestamp = snap_content["Single-End"]["timestamp"] # delete the timestamp in json @@ -103,7 +103,7 @@ def test_update_snapshot_module(self): ) snap_generator.run() - with open(snap_path, "r") as fh: + with open(snap_path) as fh: snap_content = json.load(fh) assert "Single-End" in snap_content assert snap_content["Single-End"]["timestamp"] != original_timestamp diff --git a/tests/lint/actions_awsfulltest.py b/tests/lint/actions_awsfulltest.py index 30293e31a4..bbda92a4d1 100644 --- a/tests/lint/actions_awsfulltest.py +++ b/tests/lint/actions_awsfulltest.py @@ -19,7 +19,7 @@ def test_actions_awsfulltest_pass(self): # Edit .github/workflows/awsfulltest.yml to use -profile test_full new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml"), "r") as fh: + with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml")) as fh: awsfulltest_yml = fh.read() awsfulltest_yml = awsfulltest_yml.replace("-profile test ", "-profile test_full ") with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml"), "w") as fh: @@ -44,7 +44,7 @@ def test_actions_awsfulltest_fail(self): # Edit .github/workflows/awsfulltest.yml to use -profile test_full new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml"), "r") as fh: + with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml")) as fh: awsfulltest_yml = yaml.safe_load(fh) del awsfulltest_yml[True]["release"] with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml"), "w") as fh: diff --git a/tests/lint/actions_awstest.py b/tests/lint/actions_awstest.py index 0e19f781aa..7bfa6052f8 100644 --- a/tests/lint/actions_awstest.py +++ b/tests/lint/actions_awstest.py @@ -20,7 +20,7 @@ def test_actions_awstest_fail(self): # Edit .github/workflows/awsfulltest.yml to use -profile test_full new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml[True]["push"] = ["master"] with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: diff --git a/tests/lint/actions_ci.py b/tests/lint/actions_ci.py index d44dbb73b5..8734b2f78b 100644 --- a/tests/lint/actions_ci.py +++ b/tests/lint/actions_ci.py @@ -31,7 +31,7 @@ def test_actions_ci_fail_wrong_trigger(self): # Edit .github/workflows/actions_ci.yml to mess stuff up! new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "ci.yml"), "r") as fh: + with open(os.path.join(new_pipeline, ".github", "workflows", "ci.yml")) as fh: ci_yml = yaml.safe_load(fh) ci_yml[True]["push"] = ["dev", "patch"] ci_yml["jobs"]["test"]["strategy"]["matrix"] = {"nxf_versionnn": ["foo", ""]} diff --git a/tests/lint/actions_schema_validation.py b/tests/lint/actions_schema_validation.py index 48bb07e4dd..ad65d90018 100644 --- a/tests/lint/actions_schema_validation.py +++ b/tests/lint/actions_schema_validation.py @@ -9,7 +9,7 @@ def test_actions_schema_validation_missing_jobs(self): """Missing 'jobs' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml.pop("jobs") with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: @@ -27,7 +27,7 @@ def test_actions_schema_validation_missing_on(self): """Missing 'on' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml.pop(True) with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: @@ -46,7 +46,7 @@ def test_actions_schema_validation_fails_for_additional_property(self): """Missing 'jobs' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml["not_jobs"] = awstest_yml["jobs"] with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: diff --git a/tests/lint/merge_markers.py b/tests/lint/merge_markers.py index be0d076757..64a62e25c3 100644 --- a/tests/lint/merge_markers.py +++ b/tests/lint/merge_markers.py @@ -7,7 +7,7 @@ def test_merge_markers_found(self): """Missing 'jobs' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, "main.nf"), "r") as fh: + with open(os.path.join(new_pipeline, "main.nf")) as fh: main_nf_content = fh.read() main_nf_content = ">>>>>>>\n" + main_nf_content with open(os.path.join(new_pipeline, "main.nf"), "w") as fh: diff --git a/tests/lint/multiqc_config.py b/tests/lint/multiqc_config.py index 446b4378b0..721560ce81 100644 --- a/tests/lint/multiqc_config.py +++ b/tests/lint/multiqc_config.py @@ -18,7 +18,7 @@ def test_multiqc_config_exists_ignore(self): def test_multiqc_config_missing_report_section_order(self): """Test that linting fails if the multiqc_config.yml file is missing the report_section_order""" new_pipeline = self._make_pipeline_copy() - with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "r") as fh: + with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) mqc_yml_tmp = mqc_yml mqc_yml.pop("report_section_order") @@ -36,7 +36,7 @@ def test_multiqc_config_missing_report_section_order(self): def test_multiqc_incorrect_export_plots(self): """Test that linting fails if the multiqc_config.yml file has an incorrect value for export_plots""" new_pipeline = self._make_pipeline_copy() - with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "r") as fh: + with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) mqc_yml_tmp = mqc_yml mqc_yml["export_plots"] = False @@ -54,7 +54,7 @@ def test_multiqc_incorrect_export_plots(self): def test_multiqc_config_report_comment_fail(self): """Test that linting fails if the multiqc_config.yml file has an incorrect report_comment""" new_pipeline = self._make_pipeline_copy() - with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "r") as fh: + with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) mqc_yml_tmp = mqc_yml mqc_yml["report_comment"] = "This is a test" @@ -73,7 +73,7 @@ def test_multiqc_config_report_comment_fail(self): def test_multiqc_config_report_comment_release_fail(self): """Test that linting fails if the multiqc_config.yml file has an incorrect report_comment for a release version""" new_pipeline = self._make_pipeline_copy() - with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "r") as fh: + with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) mqc_yml_tmp = mqc_yml with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: diff --git a/tests/lint/nextflow_config.py b/tests/lint/nextflow_config.py index 1542b8cf65..5d5f8e7345 100644 --- a/tests/lint/nextflow_config.py +++ b/tests/lint/nextflow_config.py @@ -43,7 +43,7 @@ def test_nextflow_config_missing_test_profile_failed(self): new_pipeline = self._make_pipeline_copy() # Change the name of the test profile so there is no such profile nf_conf_file = os.path.join(new_pipeline, "nextflow.config") - with open(nf_conf_file, "r") as f: + with open(nf_conf_file) as f: content = f.read() fail_content = re.sub(r"\btest\b", "testfail", content) with open(nf_conf_file, "w") as f: diff --git a/tests/modules/bump_versions.py b/tests/modules/bump_versions.py index e71c388c47..ac227b92ae 100644 --- a/tests/modules/bump_versions.py +++ b/tests/modules/bump_versions.py @@ -11,7 +11,7 @@ def test_modules_bump_versions_single_module(self): """Test updating a single module""" # Change the bpipe/test version to an older version env_yml_path = os.path.join(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml") - with open(env_yml_path, "r") as fh: + with open(env_yml_path) as fh: content = fh.read() new_content = re.sub(r"bioconda::star=\d.\d.\d\D?", r"bioconda::star=2.6.1d", content) with open(env_yml_path, "w") as fh: @@ -40,7 +40,7 @@ def test_modules_bump_versions_fail_unknown_version(self): """Fail because of an unknown version""" # Change the bpipe/test version to an older version env_yml_path = os.path.join(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml") - with open(env_yml_path, "r") as fh: + with open(env_yml_path) as fh: content = fh.read() new_content = re.sub(r"bioconda::bpipe=\d.\d.\d\D?", r"bioconda::bpipe=xxx", content) with open(env_yml_path, "w") as fh: diff --git a/tests/modules/create.py b/tests/modules/create.py index 29b027421a..460a1439cb 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -86,9 +86,9 @@ def test_modules_migrate(self, mock_rich_ask): # Clone modules repo with pytests shutil.rmtree(self.nfcore_modules) Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) - with open(module_dir / "main.nf", "r") as fh: + with open(module_dir / "main.nf") as fh: old_main_nf = fh.read() - with open(module_dir / "meta.yml", "r") as fh: + with open(module_dir / "meta.yml") as fh: old_meta_yml = fh.read() # Create a module with --migrate-pytest @@ -96,9 +96,9 @@ def test_modules_migrate(self, mock_rich_ask): module_create = nf_core.modules.ModuleCreate(self.nfcore_modules, "samtools/sort", migrate_pytest=True) module_create.create() - with open(module_dir / "main.nf", "r") as fh: + with open(module_dir / "main.nf") as fh: new_main_nf = fh.read() - with open(module_dir / "meta.yml", "r") as fh: + with open(module_dir / "meta.yml") as fh: new_meta_yml = fh.read() nextflow_config = module_dir / "tests" / "nextflow.config" diff --git a/tests/modules/lint.py b/tests/modules/lint.py index a8a775e6f6..a5d8567b76 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -333,7 +333,7 @@ def test_modules_lint_snapshot_file_missing_fail(self): def test_modules_lint_snapshot_file_not_needed(self): """Test linting a module which doesn't need a snapshot file by removing the snapshot keyword in the main.nf.test file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test"), "r") as fh: + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test")) as fh: content = fh.read() new_content = content.replace("snapshot(", "snap (") with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test"), "w") as fh: @@ -372,7 +372,7 @@ def test_modules_environment_yml_file_sorted_correctly(self): def test_modules_environment_yml_file_sorted_incorrectly(self): """Test linting a module with an incorrectly sorted environment.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "r") as fh: + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml")) as fh: yaml_content = yaml.safe_load(fh) # Add a new dependency to the environment.yml file and reverse the order yaml_content["dependencies"].append("z") @@ -548,7 +548,7 @@ def test_modules_missing_test_main_nf(self): def test_modules_missing_required_tag(self): """Test linting a module with a missing required tag""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test"), "r") as fh: + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test")) as fh: content = fh.read() new_content = content.replace("modules_nfcore", "foo") with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test"), "w") as fh: @@ -581,7 +581,7 @@ def test_modules_missing_tags_yml(self): def test_modules_incorrect_tags_yml_key(self): """Test linting a module with an incorrect key in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "r") as fh: + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: content = fh.read() new_content = content.replace("bpipe/test:", "bpipe_test:") with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: @@ -598,7 +598,7 @@ def test_modules_incorrect_tags_yml_key(self): def test_modules_incorrect_tags_yml_values(self): """Test linting a module with an incorrect path in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "r") as fh: + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: content = fh.read() new_content = content.replace("modules/nf-core/bpipe/test/**", "foo") with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 63ee4e743d..a054b6b131 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -17,7 +17,7 @@ def test_get_modules_json(self): """Checks that the get_modules_json function returns the correct result""" mod_json_path = os.path.join(self.pipeline_dir, "modules.json") - with open(mod_json_path, "r") as fh: + with open(mod_json_path) as fh: try: mod_json_sb = json.load(fh) except json.JSONDecodeError as e: @@ -73,7 +73,7 @@ def test_mod_json_create(self): def modify_main_nf(path): """Modify a file to test patch creation""" - with open(path, "r") as fh: + with open(path) as fh: lines = fh.readlines() # Modify $meta.id to $meta.single_end lines[1] = ' tag "$meta.single_end"\n' @@ -112,7 +112,7 @@ def test_mod_json_create_with_patch(self): assert "branch" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["fastqc"] # Check that fastqc/main.nf maintains the changes - with open(module_path / "main.nf", "r") as fh: + with open(module_path / "main.nf") as fh: lines = fh.readlines() assert lines[1] == ' tag "$meta.single_end"\n' @@ -214,7 +214,7 @@ def test_mod_json_dump(self): assert os.path.exists(mod_json_path) # Check that the dump function writes the correct content - with open(mod_json_path, "r") as f: + with open(mod_json_path) as f: try: mod_json_new = json.load(f) except json.JSONDecodeError as e: diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 338d890f2f..dc939c7ea7 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -43,7 +43,7 @@ def setup_patch(pipeline_dir, modify_module): def modify_main_nf(path): """Modify a file to test patch creation""" - with open(path, "r") as fh: + with open(path) as fh: lines = fh.readlines() # We want a patch file that looks something like: # - tuple val(meta), path(reads) @@ -99,7 +99,7 @@ def test_create_patch_change(self): ) # Check that the correct lines are in the patch file - with open(module_path / patch_fn, "r") as fh: + with open(module_path / patch_fn) as fh: patch_lines = fh.readlines() module_relpath = module_path.relative_to(self.pipeline_dir) assert f"--- {module_relpath / 'main.nf'}\n" in patch_lines, module_relpath / "main.nf" @@ -157,7 +157,7 @@ def test_create_patch_try_apply_successful(self): ) # Check that the correct lines are in the patch file - with open(module_path / patch_fn, "r") as fh: + with open(module_path / patch_fn) as fh: patch_lines = fh.readlines() module_relpath = module_path.relative_to(self.pipeline_dir) assert f"--- {module_relpath / 'main.nf'}\n" in patch_lines @@ -167,7 +167,7 @@ def test_create_patch_try_apply_successful(self): assert "+ tuple val(meta), path(reads), path(index)\n" in patch_lines # Check that 'main.nf' is updated correctly - with open(module_path / "main.nf", "r") as fh: + with open(module_path / "main.nf") as fh: main_nf_lines = fh.readlines() # These lines should have been removed by the patch assert " tuple val(meta), path(reads)\n" not in main_nf_lines @@ -258,7 +258,7 @@ def test_create_patch_update_success(self): ), modules_json_obj.get_patch_fn(BISMARK_ALIGN, GITLAB_URL, REPO_NAME) # Check that the correct lines are in the patch file - with open(module_path / patch_fn, "r") as fh: + with open(module_path / patch_fn) as fh: patch_lines = fh.readlines() module_relpath = module_path.relative_to(self.pipeline_dir) assert f"--- {module_relpath / 'main.nf'}\n" in patch_lines @@ -268,7 +268,7 @@ def test_create_patch_update_success(self): assert "+ tuple val(meta), path(reads), path(index)\n" in patch_lines # Check that 'main.nf' is updated correctly - with open(module_path / "main.nf", "r") as fh: + with open(module_path / "main.nf") as fh: main_nf_lines = fh.readlines() # These lines should have been removed by the patch assert " tuple val(meta), path(reads)\n" not in main_nf_lines @@ -300,7 +300,7 @@ def test_create_patch_update_fail(self): ) # Save the file contents for downstream comparison - with open(module_path / patch_fn, "r") as fh: + with open(module_path / patch_fn) as fh: patch_contents = fh.read() update_obj = nf_core.modules.ModuleUpdate( @@ -317,14 +317,14 @@ def test_create_patch_update_fail(self): temp_module_dir = temp_dir / BISMARK_ALIGN for file in os.listdir(temp_module_dir): assert file in os.listdir(module_path) - with open(module_path / file, "r") as fh: + with open(module_path / file) as fh: installed = fh.read() - with open(temp_module_dir / file, "r") as fh: + with open(temp_module_dir / file) as fh: shouldbe = fh.read() assert installed == shouldbe # Check that the patch file is unaffected - with open(module_path / patch_fn, "r") as fh: + with open(module_path / patch_fn) as fh: new_patch_contents = fh.read() assert patch_contents == new_patch_contents diff --git a/tests/modules/update.py b/tests/modules/update.py index 399e9cc12c..5208070fa5 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -345,7 +345,7 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): # We modify fastqc because it's one of the modules that can be updated and there's another one before it (custom/dumpsoftwareversions) module_path = Path(self.pipeline_dir, "modules", "nf-core", "fastqc") main_path = Path(module_path, "main.nf") - with open(main_path, "r") as fh: + with open(main_path) as fh: lines = fh.readlines() for line_index in range(len(lines)): if lines[line_index] == " label 'process_medium'\n": diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index 88d889ef20..002b889671 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -52,9 +52,9 @@ def test_subworkflows_migrate(self, mock_rich_ask): # Clone modules repo with pytests shutil.rmtree(self.nfcore_modules) Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) - with open(subworkflow_dir / "main.nf", "r") as fh: + with open(subworkflow_dir / "main.nf") as fh: old_main_nf = fh.read() - with open(subworkflow_dir / "meta.yml", "r") as fh: + with open(subworkflow_dir / "meta.yml") as fh: old_meta_yml = fh.read() # Create a subworkflow with --migrate-pytest @@ -64,9 +64,9 @@ def test_subworkflows_migrate(self, mock_rich_ask): ) subworkflow_create.create() - with open(subworkflow_dir / "main.nf", "r") as fh: + with open(subworkflow_dir / "main.nf") as fh: new_main_nf = fh.read() - with open(subworkflow_dir / "meta.yml", "r") as fh: + with open(subworkflow_dir / "meta.yml") as fh: new_meta_yml = fh.read() nextflow_config = subworkflow_dir / "tests" / "nextflow.config" diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 94c27b60ab..b53fef7f0e 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -86,9 +86,7 @@ def test_subworkflows_lint_snapshot_file_missing_fail(self): def test_subworkflows_lint_snapshot_file_not_needed(self): """Test linting a subworkflow which doesn't need a snapshot file by removing the snapshot keyword in the main.nf.test file""" - with open( - Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test"), "r" - ) as fh: + with open(Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test")) as fh: content = fh.read() new_content = content.replace("snapshot(", "snap (") with open( diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index f9cc89bd43..32a69ba180 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -72,7 +72,7 @@ def test_install_at_hash_and_update_and_save_diff_to_file(self): assert update_obj.update("fastq_align_bowtie2") is True assert cmp_component(tmpdir, sw_path) is True - with open(patch_path, "r") as fh: + with open(patch_path) as fh: line = fh.readline() assert line.startswith( "Changes in module 'nf-core/fastq_align_bowtie2' between (f3c078809a2513f1c95de14f6633fe1f03572fdb) and" diff --git a/tests/test_launch.py b/tests/test_launch.py index 4a07b255d9..dc8d6b147c 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -234,7 +234,7 @@ def test_ob_to_questionary_number(self): assert result["validate"]("") is True assert result["validate"]("123.56.78") == "Must be a number" assert result["validate"]("123.56sdkfjb") == "Must be a number" - assert result["filter"]("123.456") == float(123.456) + assert result["filter"]("123.456") == 123.456 assert result["filter"]("") == "" def test_ob_to_questionary_integer(self): @@ -248,7 +248,7 @@ def test_ob_to_questionary_integer(self): assert result["validate"]("") is True assert result["validate"]("123.45") == "Must be an integer" assert result["validate"]("123.56sdkfjb") == "Must be an integer" - assert result["filter"]("123") == int(123) + assert result["filter"]("123") == 123 assert result["filter"]("") == "" def test_ob_to_questionary_range(self): @@ -321,7 +321,7 @@ def test_build_command_params(self): == f'nextflow run {self.pipeline_dir} -params-file "{os.path.relpath(self.nf_params_fn)}"' ) # Check saved parameters file - with open(self.nf_params_fn, "r") as fh: + with open(self.nf_params_fn) as fh: try: saved_json = json.load(fh) except json.JSONDecodeError as e: diff --git a/tests/test_lint.py b/tests/test_lint.py index 3ed069ca8b..81c7d74ec6 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -134,7 +134,7 @@ def test_json_output(self, tmp_dir): self.lint_obj._save_json_results(json_fn) # Load created JSON file and check its contents - with open(json_fn, "r") as fh: + with open(json_fn) as fh: try: saved_json = json.load(fh) except json.JSONDecodeError as e: diff --git a/tests/test_modules.py b/tests/test_modules.py index 92c8dfda3f..f7ada2a483 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -47,7 +47,7 @@ def create_modules_repo_dummy(tmp_dir): # Remove doi from meta.yml which makes lint fail meta_yml_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "meta.yml") - with open(meta_yml_path, "r") as fh: + with open(meta_yml_path) as fh: meta_yml = yaml.safe_load(fh) del meta_yml["tools"][0]["bpipe"]["doi"] with open(meta_yml_path, "w") as fh: @@ -60,7 +60,7 @@ def create_modules_repo_dummy(tmp_dir): # remove "TODO" statements from main.nf main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") - with open(main_nf_path, "r") as fh: + with open(main_nf_path) as fh: main_nf = fh.read() main_nf = main_nf.replace("TODO", "") with open(main_nf_path, "w") as fh: @@ -68,7 +68,7 @@ def create_modules_repo_dummy(tmp_dir): # remove "TODO" statements from main.nf.test main_nf_test_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test") - with open(main_nf_test_path, "r") as fh: + with open(main_nf_test_path) as fh: main_nf_test = fh.read() main_nf_test = main_nf_test.replace("TODO", "") with open(main_nf_test_path, "w") as fh: diff --git a/tests/test_params_file.py b/tests/test_params_file.py index c6aeab0e08..13c82f5188 100644 --- a/tests/test_params_file.py +++ b/tests/test_params_file.py @@ -31,7 +31,7 @@ def setup_class(cls): cls.invalid_template_schema = os.path.join(cls.template_dir, "nextflow_schema_invalid.json") # Remove the allOf section to make the schema invalid - with open(cls.template_schema, "r") as fh: + with open(cls.template_schema) as fh: o = json.load(fh) del o["allOf"] @@ -49,7 +49,7 @@ def test_build_template(self): assert os.path.exists(outfile) - with open(outfile, "r") as fh: + with open(outfile) as fh: out = fh.read() assert "nf-core/testpipeline" in out From d00c1b6dc257e870c83b8ac52207a28c987b8cbb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 4 Jan 2024 10:36:22 +0100 Subject: [PATCH 18/22] manually fix other ruff errors --- nf_core/__main__.py | 16 +++++------ nf_core/components/lint/__init__.py | 2 +- nf_core/components/nfcore_component.py | 4 +-- nf_core/download.py | 28 +++++++++---------- nf_core/modules/__init__.py | 2 +- nf_core/modules/bump_versions.py | 8 ++++-- nf_core/modules/lint/__init__.py | 6 ++-- nf_core/modules/modules_utils.py | 4 +-- nf_core/subworkflows/lint/__init__.py | 6 ++-- nf_core/sync.py | 38 +++++++++++++------------- nf_core/utils.py | 14 +++++----- pyproject.toml | 4 +++ tests/modules/bump_versions.py | 4 +-- tests/test_download.py | 16 +++++------ tests/test_lint.py | 2 +- tests/test_sync.py | 18 ++++++------ 16 files changed, 89 insertions(+), 83 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 8cfacf7399..194b2030be 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -934,7 +934,7 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, Test modules within a pipeline or a clone of the nf-core/modules repository. """ - from nf_core.components.lint import LintException + from nf_core.components.lint import LintExceptionError from nf_core.modules import ModuleLint try: @@ -960,7 +960,7 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, ) if len(module_lint.failed) > 0: sys.exit(1) - except LintException as e: + except LintExceptionError as e: log.error(e) sys.exit(1) except (UserWarning, LookupError) as e: @@ -1020,7 +1020,7 @@ def bump_versions(ctx, tool, dir, all, show_all): the nf-core/modules repo. """ from nf_core.modules.bump_versions import ModuleVersionBumper - from nf_core.modules.modules_utils import ModuleException + from nf_core.modules.modules_utils import ModuleExceptionError try: version_bumper = ModuleVersionBumper( @@ -1030,7 +1030,7 @@ def bump_versions(ctx, tool, dir, all, show_all): ctx.obj["modules_repo_no_pull"], ) version_bumper.bump_versions(module=tool, all_modules=all, show_uptodate=show_all) - except ModuleException as e: + except ModuleExceptionError as e: log.error(e) sys.exit(1) except (UserWarning, LookupError) as e: @@ -1207,7 +1207,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo Test subworkflows within a pipeline or a clone of the nf-core/modules repository. """ - from nf_core.components.lint import LintException + from nf_core.components.lint import LintExceptionError from nf_core.subworkflows import SubworkflowLint try: @@ -1232,7 +1232,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo ) if len(subworkflow_lint.failed) > 0: sys.exit(1) - except LintException as e: + except LintExceptionError as e: log.error(e) sys.exit(1) except (UserWarning, LookupError) as e: @@ -1647,7 +1647,7 @@ def sync(dir, from_branch, pull_request, github_repository, username, template_y the pipeline. It is run automatically for all pipelines when ever a new release of [link=https://github.com/nf-core/tools]nf-core/tools[/link] (and the included template) is made. """ - from nf_core.sync import PipelineSync, PullRequestException, SyncException + from nf_core.sync import PipelineSync, PullRequestExceptionError, SyncExceptionError from nf_core.utils import is_pipeline_directory # Check if pipeline directory contains necessary files @@ -1657,7 +1657,7 @@ def sync(dir, from_branch, pull_request, github_repository, username, template_y sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml) try: sync_obj.sync() - except (SyncException, PullRequestException) as e: + except (SyncExceptionError, PullRequestExceptionError) as e: log.error(e) sys.exit(1) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index d4867a7067..3c2fb9dde3 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -26,7 +26,7 @@ log = logging.getLogger(__name__) -class LintException(Exception): +class LintExceptionError(Exception): """Exception raised when there was an error with module or subworkflow linting""" pass diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index e7e12c7837..121b83490b 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -168,7 +168,7 @@ def get_inputs_from_main_nf(self): input_data = data.split("input:")[1].split("output:")[0] regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" matches = re.finditer(regex, input_data, re.MULTILINE) - for matchNum, match in enumerate(matches, start=1): + for _, match in enumerate(matches, start=1): if match.group(3): inputs.append(match.group(3)) elif match.group(4): @@ -187,7 +187,7 @@ def get_outputs_from_main_nf(self): output_data = data.split("output:")[1].split("when:")[0] regex = r"emit:\s*([^)\s,]+)" matches = re.finditer(regex, output_data, re.MULTILINE) - for matchNum, match in enumerate(matches, start=1): + for _, match in enumerate(matches, start=1): outputs.append(match.group(1)) log.info(f"Found {len(outputs)} outputs in {self.main_nf}") self.outputs = outputs diff --git a/nf_core/download.py b/nf_core/download.py index 86de160858..4c0bc97f42 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -20,7 +20,7 @@ import rich import rich.progress from git.exc import GitCommandError, InvalidGitRepositoryError -from pkg_resources import parse_version as VersionParser +from pkg_resources import parse_version as version_parser import nf_core import nf_core.list @@ -1065,10 +1065,10 @@ def get_singularity_images(self, current_revision=""): self.singularity_pull_image(*container, library, progress) # Pulling the image was successful, no ContainerError was raised, break the library loop break - except ContainerError.ImageExists: + except ContainerError.ImageExistsError: # Pulling not required break - except ContainerError.RegistryNotFound as e: + except ContainerError.RegistryNotFoundError as e: self.container_library.remove(library) # The only library was removed if not self.container_library: @@ -1078,13 +1078,13 @@ def get_singularity_images(self, current_revision=""): else: # Other libraries can be used continue - except ContainerError.ImageNotFound as e: + except ContainerError.ImageNotFoundError as e: # Try other registries if e.error_log.absolute_URI: break # there no point in trying other registries if absolute URI was specified. else: continue - except ContainerError.InvalidTag: + except ContainerError.InvalidTagError: # Try other registries continue except ContainerError.OtherError as e: @@ -1523,7 +1523,7 @@ def tidy_tags_and_branches(self): else: # desired revisions may contain arbitrary branch names that do not correspond to valid sematic versioning patterns. valid_versions = [ - VersionParser(v) + version_parser(v) for v in desired_revisions if re.match(r"\d+\.\d+(?:\.\d+)*(?:[\w\-_])*", v) ] @@ -1582,7 +1582,7 @@ def __init__(self, container, registry, address, absolute_URI, out_path, singula for line in error_msg: if re.search(r"dial\stcp.*no\ssuch\shost", line): - self.error_type = self.RegistryNotFound(self) + self.error_type = self.RegistryNotFoundError(self) break elif ( re.search(r"requested\saccess\sto\sthe\sresource\sis\sdenied", line) @@ -1594,13 +1594,13 @@ def __init__(self, container, registry, address, absolute_URI, out_path, singula # unauthorized: authentication required # Quay.io: StatusCode: 404, \n'] # ghcr.io: Requesting bearer token: invalid status code from registry 400 (Bad Request) - self.error_type = self.ImageNotFound(self) + self.error_type = self.ImageNotFoundError(self) break elif re.search(r"manifest\sunknown", line): - self.error_type = self.InvalidTag(self) + self.error_type = self.InvalidTagError(self) break elif re.search(r"Image\sfile\salready\sexists", line): - self.error_type = self.ImageExists(self) + self.error_type = self.ImageExistsError(self) break else: continue @@ -1614,7 +1614,7 @@ def __init__(self, container, registry, address, absolute_URI, out_path, singula raise self.error_type - class RegistryNotFound(ConnectionRefusedError): + class RegistryNotFoundError(ConnectionRefusedError): """The specified registry does not resolve to a valid IP address""" def __init__(self, error_log): @@ -1627,7 +1627,7 @@ def __init__(self, error_log): ) super().__init__(self.message, self.helpmessage, self.error_log) - class ImageNotFound(FileNotFoundError): + class ImageNotFoundError(FileNotFoundError): """The image can not be found in the registry""" def __init__(self, error_log): @@ -1643,7 +1643,7 @@ def __init__(self, error_log): super().__init__(self.message) - class InvalidTag(AttributeError): + class InvalidTagError(AttributeError): """Image and registry are valid, but the (version) tag is not""" def __init__(self, error_log): @@ -1652,7 +1652,7 @@ def __init__(self, error_log): self.helpmessage = f'Please chose a different library than {self.error_log.registry}\nor try to locate the "{self.error_log.address.split(":")[-1]}" version of "{self.error_log.container}" manually.\nPlease troubleshoot the command \n"{" ".join(self.error_log.singularity_command)}" manually.\n' super().__init__(self.message) - class ImageExists(FileExistsError): + class ImageExistsError(FileExistsError): """Image already exists in cache/output directory.""" def __init__(self, error_log): diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index 4b36f302bd..6be871ece8 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -6,7 +6,7 @@ from .list import ModuleList from .modules_json import ModulesJson from .modules_repo import ModulesRepo -from .modules_utils import ModuleException +from .modules_utils import ModuleExceptionError from .patch import ModulePatch from .remove import ModuleRemove from .update import ModuleUpdate diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 75ec15c206..b9003be974 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -71,7 +71,7 @@ def bump_versions( # Verify that this is not a pipeline if not self.repo_type == "modules": - raise nf_core.modules.modules_utils.ModuleException( + raise nf_core.modules.modules_utils.ModuleExceptionError( "This command only works on the nf-core/modules repository, not on pipelines!" ) @@ -102,12 +102,14 @@ def bump_versions( if module: self.show_up_to_date = True if all_modules: - raise nf_core.modules.modules_utils.ModuleException( + raise nf_core.modules.modules_utils.ModuleExceptionError( "You cannot specify a tool and request all tools to be bumped." ) nfcore_modules = [m for m in nfcore_modules if m.component_name == module] if len(nfcore_modules) == 0: - raise nf_core.modules.modules_utils.ModuleException(f"Could not find the specified module: '{module}'") + raise nf_core.modules.modules_utils.ModuleExceptionError( + f"Could not find the specified module: '{module}'" + ) progress_bar = Progress( "[bold blue]{task.description}", diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 75d2d830b8..866e6312aa 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -15,7 +15,7 @@ import nf_core.modules.modules_utils import nf_core.utils -from nf_core.components.lint import ComponentLint, LintException, LintResult +from nf_core.components.lint import ComponentLint, LintExceptionError, LintResult from nf_core.lint_utils import console log = logging.getLogger(__name__) @@ -118,11 +118,11 @@ def lint( # Only lint the given module if module: if all_modules: - raise LintException("You cannot specify a tool and request all tools to be linted.") + raise LintExceptionError("You cannot specify a tool and request all tools to be linted.") local_modules = [] remote_modules = [m for m in self.all_remote_components if m.component_name == module] if len(remote_modules) == 0: - raise LintException(f"Could not find the specified module: '{module}'") + raise LintExceptionError(f"Could not find the specified module: '{module}'") else: local_modules = self.all_local_components remote_modules = self.all_remote_components diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index ecc3e1605b..ca8993483b 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -9,7 +9,7 @@ log = logging.getLogger(__name__) -class ModuleException(Exception): +class ModuleExceptionError(Exception): """Exception raised when there was an error with module commands""" pass @@ -69,7 +69,7 @@ def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], Lis if os.path.exists(nfcore_modules_dir): for m in sorted([m for m in os.listdir(nfcore_modules_dir) if not m == "lib"]): if not os.path.isdir(os.path.join(nfcore_modules_dir, m)): - raise ModuleException( + raise ModuleExceptionError( f"File found in '{nfcore_modules_dir}': '{m}'! This directory should only contain module directories." ) m_content = os.listdir(os.path.join(nfcore_modules_dir, m)) diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index dc8fa68c30..3a87190422 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -15,7 +15,7 @@ import nf_core.modules.modules_utils import nf_core.utils -from nf_core.components.lint import ComponentLint, LintException, LintResult +from nf_core.components.lint import ComponentLint, LintExceptionError, LintResult from nf_core.lint_utils import console log = logging.getLogger(__name__) @@ -113,11 +113,11 @@ def lint( # Only lint the given module if subworkflow: if all_subworkflows: - raise LintException("You cannot specify a tool and request all tools to be linted.") + raise LintExceptionError("You cannot specify a tool and request all tools to be linted.") local_subworkflows = [] remote_subworkflows = [s for s in self.all_remote_components if s.component_name == subworkflow] if len(remote_subworkflows) == 0: - raise LintException(f"Could not find the specified subworkflow: '{subworkflow}'") + raise LintExceptionError(f"Could not find the specified subworkflow: '{subworkflow}'") else: local_subworkflows = self.all_local_components remote_subworkflows = self.all_remote_components diff --git a/nf_core/sync.py b/nf_core/sync.py index 63614c98c9..995baeacd2 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -23,13 +23,13 @@ log = logging.getLogger(__name__) -class SyncException(Exception): +class SyncExceptionError(Exception): """Exception raised when there was an error with TEMPLATE branch synchronisation""" pass -class PullRequestException(Exception): +class PullRequestExceptionError(Exception): """Exception raised when there was an error creating a Pull-Request on GitHub.com""" pass @@ -138,20 +138,20 @@ def sync(self): try: # Check that we have an API auth token if os.environ.get("GITHUB_AUTH_TOKEN", "") == "": - raise PullRequestException("GITHUB_AUTH_TOKEN not set!") + raise PullRequestExceptionError("GITHUB_AUTH_TOKEN not set!") # Check that we know the github username and repo name if self.gh_username is None and self.gh_repo is None: - raise PullRequestException("Could not find GitHub username and repo name") + raise PullRequestExceptionError("Could not find GitHub username and repo name") self.push_template_branch() self.create_merge_base_branch() self.push_merge_branch() self.make_pull_request() self.close_open_template_merge_prs() - except PullRequestException as e: + except PullRequestExceptionError as e: self.reset_target_dir() - raise PullRequestException(e) + raise PullRequestExceptionError(e) self.reset_target_dir() @@ -170,7 +170,7 @@ def inspect_sync_dir(self): try: self.repo = git.Repo(self.pipeline_dir) except InvalidGitRepositoryError: - raise SyncException(f"'{self.pipeline_dir}' does not appear to be a git repository") + raise SyncExceptionError(f"'{self.pipeline_dir}' does not appear to be a git repository") # get current branch so we can switch back later self.original_branch = self.repo.active_branch.name @@ -178,7 +178,7 @@ def inspect_sync_dir(self): # Check to see if there are uncommitted changes on current branch if self.repo.is_dirty(untracked_files=True): - raise SyncException( + raise SyncExceptionError( "Uncommitted changes found in pipeline directory!\nPlease commit these before running nf-core sync" ) @@ -192,7 +192,7 @@ def get_wf_config(self): log.info(f"Checking out workflow branch '{self.from_branch}'") self.repo.git.checkout(self.from_branch) except GitCommandError: - raise SyncException(f"Branch `{self.from_branch}` not found!") + raise SyncExceptionError(f"Branch `{self.from_branch}` not found!") # If not specified, get the name of the active branch if not self.from_branch: @@ -208,7 +208,7 @@ def get_wf_config(self): # Check that we have the required variables for rvar in self.required_config_vars: if rvar not in self.wf_config: - raise SyncException(f"Workflow config variable `{rvar}` not found!") + raise SyncExceptionError(f"Workflow config variable `{rvar}` not found!") def checkout_template_branch(self): """ @@ -223,7 +223,7 @@ def checkout_template_branch(self): try: self.repo.git.checkout("TEMPLATE") except GitCommandError: - raise SyncException("Could not check out branch 'origin/TEMPLATE' or 'TEMPLATE'") + raise SyncExceptionError("Could not check out branch 'origin/TEMPLATE' or 'TEMPLATE'") def delete_template_branch_files(self): """ @@ -242,7 +242,7 @@ def delete_template_branch_files(self): elif os.path.isdir(file_path): shutil.rmtree(file_path) except Exception as e: - raise SyncException(e) + raise SyncExceptionError(e) def make_template_pipeline(self): """ @@ -272,7 +272,7 @@ def make_template_pipeline(self): except Exception as err: # Reset to where you were to prevent git getting messed up. self.repo.git.reset("--hard") - raise SyncException(f"Failed to rebuild pipeline from template with error:\n{err}") + raise SyncExceptionError(f"Failed to rebuild pipeline from template with error:\n{err}") def commit_template_changes(self): """If we have any changes with the new template files, make a git commit""" @@ -287,7 +287,7 @@ def commit_template_changes(self): self.made_changes = True log.info("Committed changes to 'TEMPLATE' branch") except Exception as e: - raise SyncException(f"Could not commit changes to TEMPLATE:\n{e}") + raise SyncExceptionError(f"Could not commit changes to TEMPLATE:\n{e}") return True def push_template_branch(self): @@ -299,7 +299,7 @@ def push_template_branch(self): try: self.repo.git.push() except GitCommandError as e: - raise PullRequestException(f"Could not push TEMPLATE branch:\n {e}") + raise PullRequestExceptionError(f"Could not push TEMPLATE branch:\n {e}") def create_merge_base_branch(self): """Create a new branch from the updated TEMPLATE branch @@ -326,7 +326,7 @@ def create_merge_base_branch(self): try: self.repo.create_head(self.merge_branch) except GitCommandError as e: - raise SyncException(f"Could not create new branch '{self.merge_branch}'\n{e}") + raise SyncExceptionError(f"Could not create new branch '{self.merge_branch}'\n{e}") def push_merge_branch(self): """Push the newly created merge branch to the remote repository""" @@ -335,7 +335,7 @@ def push_merge_branch(self): origin = self.repo.remote() origin.push(self.merge_branch) except GitCommandError as e: - raise PullRequestException(f"Could not push branch '{self.merge_branch}':\n {e}") + raise PullRequestExceptionError(f"Could not push branch '{self.merge_branch}':\n {e}") def make_pull_request(self): """Create a pull request to a base branch (default: dev), @@ -374,7 +374,7 @@ def make_pull_request(self): ) except Exception as e: stderr.print_exception() - raise PullRequestException(f"Something went badly wrong - {e}") + raise PullRequestExceptionError(f"Something went badly wrong - {e}") else: self.gh_pr_returned_data = r.json() self.pr_url = self.gh_pr_returned_data["html_url"] @@ -462,4 +462,4 @@ def reset_target_dir(self): try: self.repo.git.checkout(self.original_branch) except GitCommandError as e: - raise SyncException(f"Could not reset to original branch `{self.original_branch}`:\n{e}") + raise SyncExceptionError(f"Could not reset to original branch `{self.original_branch}`:\n{e}") diff --git a/nf_core/utils.py b/nf_core/utils.py index 121c2de0b5..10b21018d2 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -181,9 +181,9 @@ def _load_pipeline_config(self): self.pipeline_prefix, self.pipeline_name = self.nf_config.get("manifest.name", "").strip("'").split("/") - nextflowVersionMatch = re.search(r"[0-9\.]+(-edge)?", self.nf_config.get("manifest.nextflowVersion", "")) - if nextflowVersionMatch: - self.minNextflowVersion = nextflowVersionMatch.group(0) + nextflow_version_match = re.search(r"[0-9\.]+(-edge)?", self.nf_config.get("manifest.nextflowVersion", "")) + if nextflow_version_match: + self.minNextflowVersion = nextflow_version_match.group(0) def _load_conda_environment(self): """Try to load the pipeline environment.yml file, if it exists""" @@ -312,7 +312,7 @@ def run_cmd(executable: str, cmd: str) -> Union[Tuple[bytes, bytes], None]: full_cmd = f"{executable} {cmd}" log.debug(f"Running command: {full_cmd}") try: - proc = subprocess.run(shlex.split(full_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + proc = subprocess.run(shlex.split(full_cmd), capture_output=True, check=True) return (proc.stdout, proc.stderr) except OSError as e: if e.errno == errno.ENOENT: @@ -433,7 +433,7 @@ def poll_nfcore_web_api(api_url, post_data=None): return web_response -class GitHub_API_Session(requests_cache.CachedSession): +class GitHubAPISession(requests_cache.CachedSession): """ Class to provide a single session for interacting with the GitHub API for a run. Inherits the requests_cache.CachedSession and adds additional functionality, @@ -590,7 +590,7 @@ def request_retry(self, url, post_data=None): # Single session object to use for entire codebase. Not sure if there's a better way to do this? -gh_api = GitHub_API_Session() +gh_api = GitHubAPISession() def anaconda_package(dep, dep_channels=None): @@ -792,7 +792,7 @@ def increase_indent(self, flow=False, indentless=False): See https://github.com/yaml/pyyaml/issues/234#issuecomment-765894586 """ - return super(CustomDumper, self).increase_indent(flow=flow, indentless=False) + return super().increase_indent(flow=flow, indentless=False) # HACK: insert blank lines between top-level objects # inspired by https://stackoverflow.com/a/44284819/3786245 diff --git a/pyproject.toml b/pyproject.toml index a5459fa795..d75ae89df6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,3 +23,7 @@ known-first-party = ["nf_core"] [tool.ruff.per-file-ignores] "__init__.py" = ["E402", "F401"] + +[tool.ruff.lint.pep8-naming] +extend-ignore-names = ["mocked_*", "*allOf", "*URI*"] + diff --git a/tests/modules/bump_versions.py b/tests/modules/bump_versions.py index ac227b92ae..ce8c6dbe11 100644 --- a/tests/modules/bump_versions.py +++ b/tests/modules/bump_versions.py @@ -4,7 +4,7 @@ import pytest import nf_core.modules -from nf_core.modules.modules_utils import ModuleException +from nf_core.modules.modules_utils import ModuleExceptionError def test_modules_bump_versions_single_module(self): @@ -31,7 +31,7 @@ def test_modules_bump_versions_all_modules(self): def test_modules_bump_versions_fail(self): """Fail updating a module with wrong name""" version_bumper = nf_core.modules.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) - with pytest.raises(ModuleException) as excinfo: + with pytest.raises(ModuleExceptionError) as excinfo: version_bumper.bump_versions(module="no/module") assert "Could not find the specified module:" in str(excinfo.value) diff --git a/tests/test_download.py b/tests/test_download.py index 4629779e49..7f34f7fbc6 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -258,7 +258,7 @@ def test_singularity_pull_image_singularity_installed(self, tmp_dir, mock_rich_p ) # Pull again, but now the image already exists - with pytest.raises(ContainerError.ImageExists): + with pytest.raises(ContainerError.ImageExistsError): download_obj.singularity_pull_image( "hello-world", f"{tmp_dir}/hello-world.sif", None, "docker.io", mock_rich_progress ) @@ -268,8 +268,8 @@ def test_singularity_pull_image_singularity_installed(self, tmp_dir, mock_rich_p "docker.io/bschiffthaler/sed", f"{tmp_dir}/sed.sif", None, "docker.io", mock_rich_progress ) - # try to pull from non-existing registry (Name change hello-world_new.sif is needed, otherwise ImageExists is raised before attempting to pull.) - with pytest.raises(ContainerError.RegistryNotFound): + # try to pull from non-existing registry (Name change hello-world_new.sif is needed, otherwise ImageExistsError is raised before attempting to pull.) + with pytest.raises(ContainerError.RegistryNotFoundError): download_obj.singularity_pull_image( "hello-world", f"{tmp_dir}/hello-world_new.sif", @@ -279,23 +279,23 @@ def test_singularity_pull_image_singularity_installed(self, tmp_dir, mock_rich_p ) # test Image not found for several registries - with pytest.raises(ContainerError.ImageNotFound): + with pytest.raises(ContainerError.ImageNotFoundError): download_obj.singularity_pull_image( "a-container", f"{tmp_dir}/acontainer.sif", None, "quay.io", mock_rich_progress ) - with pytest.raises(ContainerError.ImageNotFound): + with pytest.raises(ContainerError.ImageNotFoundError): download_obj.singularity_pull_image( "a-container", f"{tmp_dir}/acontainer.sif", None, "docker.io", mock_rich_progress ) - with pytest.raises(ContainerError.ImageNotFound): + with pytest.raises(ContainerError.ImageNotFoundError): download_obj.singularity_pull_image( "a-container", f"{tmp_dir}/acontainer.sif", None, "ghcr.io", mock_rich_progress ) # test Image not found for absolute URI. - with pytest.raises(ContainerError.ImageNotFound): + with pytest.raises(ContainerError.ImageNotFoundError): download_obj.singularity_pull_image( "docker.io/bschiffthaler/nothingtopullhere", f"{tmp_dir}/nothingtopullhere.sif", @@ -305,7 +305,7 @@ def test_singularity_pull_image_singularity_installed(self, tmp_dir, mock_rich_p ) # Traffic from Github Actions to GitHub's Container Registry is unlimited, so no harm should be done here. - with pytest.raises(ContainerError.InvalidTag): + with pytest.raises(ContainerError.InvalidTagError): download_obj.singularity_pull_image( "ewels/multiqc:go-rewrite", f"{tmp_dir}/umi-transfer.sif", diff --git a/tests/test_lint.py b/tests/test_lint.py index 81c7d74ec6..32913bda0d 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -58,7 +58,7 @@ def test_run_linting_function(self): and we're testing each of those individually. This is mostly to check for syntax errors.""" nf_core.lint.run_linting(self.test_pipeline_dir, False) - def test_init_PipelineLint(self): + def test_init_pipeline_lint(self): """Simply create a PipelineLint object. This checks that all of the lint test imports are working properly, diff --git a/tests/test_sync.py b/tests/test_sync.py index 597e4375d3..51a27653ab 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -44,7 +44,7 @@ def tearDown(self): def test_inspect_sync_dir_notgit(self, tmp_dir): """Try syncing an empty directory""" psync = nf_core.sync.PipelineSync(tmp_dir) - with pytest.raises(nf_core.sync.SyncException) as exc_info: + with pytest.raises(nf_core.sync.SyncExceptionError) as exc_info: psync.inspect_sync_dir() assert "does not appear to be a git repository" in exc_info.value.args[0] @@ -56,7 +56,7 @@ def test_inspect_sync_dir_dirty(self): # Try to sync, check we halt with the right error psync = nf_core.sync.PipelineSync(self.pipeline_dir) try: - with pytest.raises(nf_core.sync.SyncException) as exc_info: + with pytest.raises(nf_core.sync.SyncExceptionError) as exc_info: psync.inspect_sync_dir() assert exc_info.value.args[0].startswith("Uncommitted changes found in pipeline directory!") finally: @@ -66,7 +66,7 @@ def test_get_wf_config_no_branch(self): """Try getting a workflow config when the branch doesn't exist""" # Try to sync, check we halt with the right error psync = nf_core.sync.PipelineSync(self.pipeline_dir, from_branch="foo") - with pytest.raises(nf_core.sync.SyncException) as exc_info: + with pytest.raises(nf_core.sync.SyncExceptionError) as exc_info: psync.inspect_sync_dir() psync.get_wf_config() assert exc_info.value.args[0] == "Branch `foo` not found!" @@ -76,7 +76,7 @@ def test_get_wf_config_missing_required_config(self): # Try to sync, check we halt with the right error psync = nf_core.sync.PipelineSync(self.pipeline_dir) psync.required_config_vars = ["fakethisdoesnotexist"] - with pytest.raises(nf_core.sync.SyncException) as exc_info: + with pytest.raises(nf_core.sync.SyncExceptionError) as exc_info: psync.inspect_sync_dir() psync.get_wf_config() # Check that we did actually get some config back @@ -99,7 +99,7 @@ def test_checkout_template_branch_no_template(self): psync.repo.delete_head("TEMPLATE") - with pytest.raises(nf_core.sync.SyncException) as exc_info: + with pytest.raises(nf_core.sync.SyncExceptionError) as exc_info: psync.checkout_template_branch() assert exc_info.value.args[0] == "Could not check out branch 'origin/TEMPLATE' or 'TEMPLATE'" @@ -165,7 +165,7 @@ def test_push_template_branch_error(self): test_fn.touch() psync.commit_template_changes() # Try to push changes - with pytest.raises(nf_core.sync.PullRequestException) as exc_info: + with pytest.raises(nf_core.sync.PullRequestExceptionError) as exc_info: psync.push_template_branch() assert exc_info.value.args[0].startswith("Could not push TEMPLATE branch") @@ -220,7 +220,7 @@ def test_push_merge_branch_without_create_branch(self): psync.get_wf_config() psync.repo.create_remote("origin", self.remote_path) - with pytest.raises(nf_core.sync.PullRequestException) as exc_info: + with pytest.raises(nf_core.sync.PullRequestExceptionError) as exc_info: psync.push_merge_branch() assert exc_info.value.args[0].startswith(f"Could not push branch '{psync.merge_branch}'") @@ -329,7 +329,7 @@ def test_make_pull_request_bad_response(self, mock_post, mock_get): psync.gh_username = "bad_url" psync.gh_repo = "bad_url/response" os.environ["GITHUB_AUTH_TOKEN"] = "test" - with pytest.raises(nf_core.sync.PullRequestException) as exc_info: + with pytest.raises(nf_core.sync.PullRequestExceptionError) as exc_info: psync.make_pull_request() assert exc_info.value.args[0].startswith( "Something went badly wrong - GitHub API PR failed - got return code 404" @@ -420,6 +420,6 @@ def test_reset_target_dir_fake_branch(self): psync.original_branch = "fake_branch" - with pytest.raises(nf_core.sync.SyncException) as exc_info: + with pytest.raises(nf_core.sync.SyncExceptionError) as exc_info: psync.reset_target_dir() assert exc_info.value.args[0].startswith("Could not reset to original branch `fake_branch`") From 5df6586552ffaafdccbd6f7200ebcdc0a337a38d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 4 Jan 2024 10:40:09 +0100 Subject: [PATCH 19/22] Update .github/CONTRIBUTING.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .github/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b086b0d15e..04d327bd8c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -46,7 +46,7 @@ You can run Ruff on the command line (it's included in `requirements-dev.txt`) - ruff format . ``` -Alternatively, Black has [integrations for most common editors](https://github.com/astral-sh/ruff-lsp) and VSCode(https://github.com/astral-sh/ruff-vscode) +Alternatively, Ruff has [integrations for most common editors](https://github.com/astral-sh/ruff-lsp) and VSCode(https://github.com/astral-sh/ruff-vscode) to automatically format code when you hit save. There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if From 74a262be70347980e2dbfcf59535917b8a0c2d27 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 4 Jan 2024 10:41:17 +0100 Subject: [PATCH 20/22] fix typo --- nf_core/params_file.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/params_file.py b/nf_core/params_file.py index bfb4ff7e85..51986821b5 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -193,7 +193,7 @@ def format_param(self, name, properties, required_properties=(), show_hidden=Fal description = properties.get("description", "") self.schema_obj.get_schema_defaults() default = properties.get("default") - typ = properties.get("type") + type = properties.get("type") required = name in required_properties out += _print_wrapped(name, "-", mode="both") @@ -201,8 +201,8 @@ def format_param(self, name, properties, required_properties=(), show_hidden=Fal if description: out += _print_wrapped(description + "\n", mode="none", indent=4) - if typ: - out += _print_wrapped(f"Type: {typ}", mode="none", indent=4) + if type: + out += _print_wrapped(f"Type: {type}", mode="none", indent=4) if required: out += _print_wrapped("Required", mode="none", indent=4) From 9184c1d4b5754e41118931052ff3ee0c314a4cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 4 Jan 2024 10:59:26 +0100 Subject: [PATCH 21/22] Apply suggestions from code review --- .github/workflows/lint-code.yml | 5 ++++- nf_core/pipeline-template/.github/workflows/linting.yml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index d99241470e..0ca981cb27 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -58,9 +58,12 @@ jobs: run: | python -m pip install --upgrade pip pip install ruff - - name: Run Ruff + - name: Run Ruff check run: ruff check . + - name: Run Ruff format + run: ruff format . + # If the above check failed, post a comment on the PR explaining the failure - name: Post PR comment if: failure() diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 0911e9a7f6..03532b96de 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -51,9 +51,12 @@ jobs: run: | python -m pip install --upgrade pip pip install ruff - - name: Run Ruff + - name: Run Ruff check run: ruff check . + - name: Run Ruff format + run: ruff format . + # If the above check failed, post a comment on the PR explaining the failure - name: Post PR comment if: failure() From eb4b775a99e8f4e8d5b3d7da71dbadbf0d779562 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 4 Jan 2024 11:05:52 +0100 Subject: [PATCH 22/22] fix formatting --- .github/workflows/lint-code.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 0ca981cb27..6457d08b07 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -61,7 +61,7 @@ jobs: - name: Run Ruff check run: ruff check . - - name: Run Ruff format + - name: Run Ruff format run: ruff format . # If the above check failed, post a comment on the PR explaining the failure diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 03532b96de..164dedc2f1 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -54,7 +54,7 @@ jobs: - name: Run Ruff check run: ruff check . - - name: Run Ruff format + - name: Run Ruff format run: ruff format . # If the above check failed, post a comment on the PR explaining the failure