From 228cc2575cfe0de74c74f2a151baab0835e31b24 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 14 Apr 2023 13:14:10 -0400 Subject: [PATCH] Revert "Log error when multiple remotes have the same org name" --- .github/workflows/create-lint-wf.yml | 16 ++--------- CHANGELOG.md | 1 - nf_core/components/install.py | 23 --------------- tests/modules/install.py | 10 ------- tests/modules/lint.py | 6 ++-- tests/modules/patch.py | 43 ++++++---------------------- tests/modules/remove.py | 5 ---- tests/modules/update.py | 10 ------- tests/subworkflows/install.py | 8 ------ tests/subworkflows/list.py | 3 +- tests/test_modules.py | 6 ++-- tests/test_subworkflows.py | 6 ++-- tests/utils.py | 18 ------------ 13 files changed, 18 insertions(+), 137 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index a5b69650bf..d09bdfd822 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -107,25 +107,15 @@ jobs: - name: nf-core modules install run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force + - name: nf-core modules install gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ + - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ - name: nf-core modules list remote run: nf-core --log-file log.txt modules list remote - - name: nf-core modules remove - run: | - find nf-core-testpipeline/workflows/ -type f -exec sed -i '/^include /d' {} \; - nf-core --log-file log.txt modules remove fastqc --dir nf-core-testpipeline/ - nf-core --log-file log.txt modules remove multiqc --dir nf-core-testpipeline/ - nf-core --log-file log.txt modules remove custom/dumpsoftwareversions --dir nf-core-testpipeline/ - - - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ - - - name: nf-core modules list local gitlab - run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ - - name: nf-core modules list remote gitlab run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e59d084a8..d38c2b4af6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,6 @@ - Add the Nextflow version to Gitpod container matching the minimal Nextflow version for nf-core (according to `nextflow.config`) ([#2196](https://github.com/nf-core/tools/pull/2196)) - Use `nfcore/gitpod:dev` container in the dev branch ([#2196](https://github.com/nf-core/tools/pull/2196)) - Replace requests_mock with responses in test mocks ([#2165](https://github.com/nf-core/tools/pull/2165)). -- Add warning when installing a module from an `org_path` that exists in multiple remotes in `modules.json` ([#2228](https://github.com/nf-core/tools/pull/2228)). ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] diff --git a/nf_core/components/install.py b/nf_core/components/install.py index f7a5fe6680..850cc9d60f 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -58,12 +58,6 @@ def install(self, component, silent=False): if not silent: modules_json.check_up_to_date() - # Verify that the remote repo's org_path does not match the org_path of any alternate repo among the installed modules - if self.check_alternate_remotes(modules_json): - err_msg = f"You are trying to install {self.component_type} from different repositories with the same organization name '{self.modules_repo.repo_path}' (set in the `.nf-core.yml` file in the `org_path` field).\nThis is not supported, and will likely cause problems. org_path should be set to the github account/organization name." - log.error(err_msg) - return False - # Verify SHA if not self.modules_repo.verify_sha(self.prompt, self.sha): return False @@ -270,20 +264,3 @@ def clean_modules_json(self, component, modules_repo, modules_json): self.component_type, component, repo_to_remove, modules_repo.repo_path ) return component_values["installed_by"] - - def check_alternate_remotes(self, modules_json): - """ - Check whether there are previously installed components with the same org_path but different remote urls - Log error if multiple remotes exist. - - Return: - True: if problematic components are found - False: if problematic components are not found - """ - modules_json.load() - for repo_url, repo_content in modules_json.modules_json.get("repos", dict()).items(): - for component_type in repo_content: - for dir in repo_content.get(component_type, dict()).keys(): - if dir == self.modules_repo.repo_path and repo_url != self.modules_repo.remote_url: - return True - return False diff --git a/tests/modules/install.py b/tests/modules/install.py index fa588daf00..d01459f142 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -9,7 +9,6 @@ GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, GITLAB_URL, - remove_template_modules, with_temporary_folder, ) @@ -50,7 +49,6 @@ def test_modules_install_trimgalore_twice(self): def test_modules_install_from_gitlab(self): """Test installing a module from GitLab""" - remove_template_modules(self) assert self.mods_install_gitlab.install("fastqc") is True @@ -63,7 +61,6 @@ def test_modules_install_different_branch_fail(self): def test_modules_install_different_branch_succeed(self): """Test installing a module from a different branch""" - remove_template_modules(self) install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) # The fastp module does exists in the branch-test branch assert install_obj.install("fastp") is True @@ -86,10 +83,3 @@ def test_modules_install_tracking(self): assert mod_json["repos"]["https://github.com/nf-core/modules.git"]["modules"]["nf-core"]["trimgalore"][ "installed_by" ] == ["modules"] - - -def test_modules_install_alternate_remote(self): - """Test installing a module from a different remote with the same organization path""" - install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) - # The fastp module does exists in the branch-test branch - assert install_obj.install("fastp") is False diff --git a/tests/modules/lint.py b/tests/modules/lint.py index cc38e0ac34..18c0dc4dab 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -5,7 +5,7 @@ import nf_core.modules -from ..utils import GITLAB_URL, remove_template_modules, set_wd +from ..utils import GITLAB_URL, set_wd from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf @@ -62,7 +62,6 @@ def test_modules_lint_no_gitlab(self): def test_modules_lint_gitlab_modules(self): """Lint modules from a different remote""" - remove_template_modules(self) self.mods_install_gitlab.install("fastqc") self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) @@ -74,7 +73,7 @@ def test_modules_lint_gitlab_modules(self): def test_modules_lint_multiple_remotes(self): """Lint modules from a different remote""" - remove_template_modules(self) + self.mods_install.install("fastqc") self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) module_lint.lint(print_results=False, all_modules=True) @@ -87,7 +86,6 @@ def test_modules_lint_patched_modules(self): """ Test creating a patch file and applying it to a new version of the the files """ - remove_template_modules(self) setup_patch(self.pipeline_dir, True) # Create a patch file diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 26ca5805a5..09b892e2c8 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -7,7 +7,7 @@ import nf_core.components.components_command import nf_core.modules -from ..utils import GITLAB_URL, remove_template_modules +from ..utils import GITLAB_URL """ Test the 'nf-core modules patch' command @@ -26,7 +26,7 @@ REPO_URL = "https://gitlab.com/nf-core/modules-test.git" -def setup_patch(pipeline_dir, modify_module, pipeline_name): +def setup_patch(pipeline_dir, modify_module): install_obj = nf_core.modules.ModuleInstall( pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=ORG_SHA ) @@ -40,15 +40,6 @@ def setup_patch(pipeline_dir, modify_module, pipeline_name): modify_main_nf(module_path / "main.nf") -def modify_workflow_nf(path): - with open(path, "r") as fh: - lines = fh.readlines() - with open(path, "w") as fh: - for line in lines: - if not line.startswith("include {"): - fh.write(line) - - def modify_main_nf(path): """Modify a file to test patch creation""" with open(path, "r") as fh: @@ -69,10 +60,7 @@ def modify_main_nf(path): def test_create_patch_no_change(self): """Test creating a patch when there is no change to the module""" - # Remove modules that may cause org_path conflict - remove_template_modules(self) - - setup_patch(self.pipeline_dir, False, self.pipeline_name) + setup_patch(self.pipeline_dir, False) # Try creating a patch file patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) @@ -91,10 +79,7 @@ def test_create_patch_no_change(self): def test_create_patch_change(self): """Test creating a patch when there is a change to the module""" - # Remove modules that may cause org_path conflict - remove_template_modules(self) - - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) # Try creating a patch file patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) @@ -127,10 +112,7 @@ def test_create_patch_try_apply_successful(self): """ Test creating a patch file and applying it to a new version of the the files """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) - - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) module_path = Path(self.pipeline_dir, module_relpath) @@ -196,10 +178,7 @@ def test_create_patch_try_apply_failed(self): """ Test creating a patch file and applying it to a new version of the the files """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) - - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) module_path = Path(self.pipeline_dir, module_relpath) @@ -237,10 +216,7 @@ def test_create_patch_update_success(self): Should have the same effect as 'test_create_patch_try_apply_successful' but uses higher level api """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) - - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) # Try creating a patch file @@ -301,10 +277,7 @@ def test_create_patch_update_fail(self): """ Test creating a patch file and updating a module when there is a diff conflict """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) - - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) # Try creating a patch file diff --git a/tests/modules/remove.py b/tests/modules/remove.py index 9a8953a7ca..b343a02da7 100644 --- a/tests/modules/remove.py +++ b/tests/modules/remove.py @@ -1,7 +1,5 @@ import os -from ..utils import remove_template_modules - def test_modules_remove_trimgalore(self): """Test removing TrimGalore! module after installing it""" @@ -18,9 +16,6 @@ def test_modules_remove_trimgalore_uninstalled(self): def test_modules_remove_multiqc_from_gitlab(self): """Test removing multiqc module after installing it from an alternative source""" - # Remove modules that may cause org_path conflict - remove_template_modules(self) - self.mods_install_gitlab.install("multiqc") module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core", "multiqc") assert self.mods_remove_gitlab.remove("multiqc", force=True) diff --git a/tests/modules/update.py b/tests/modules/update.py index e28f058760..fcfd92fc39 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -24,7 +24,6 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, - remove_template_modules, ) @@ -45,7 +44,6 @@ def test_install_and_update(self): def test_install_at_hash_and_update(self): """Installs an old version of a module in the pipeline and updates it""" - remove_template_modules(self) assert self.mods_install_old.install("trimgalore") update_obj = ModuleUpdate( self.pipeline_dir, show_diff=False, update_deps=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH @@ -71,7 +69,6 @@ def test_install_at_hash_and_update(self): def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a module in the pipeline and updates it""" - remove_template_modules(self) self.mods_install_old.install("trimgalore") patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") update_obj = ModuleUpdate( @@ -112,7 +109,6 @@ def test_update_all(self): def test_update_with_config_fixed_version(self): """Try updating when there are entries in the .nf-core.yml""" - remove_template_modules(self) # Install trimgalore at the latest version assert self.mods_install_trimgalore.install("trimgalore") @@ -138,7 +134,6 @@ def test_update_with_config_fixed_version(self): def test_update_with_config_dont_update(self): """Try updating when module is to be ignored""" - remove_template_modules(self) # Install an old version of trimgalore self.mods_install_old.install("trimgalore") @@ -169,7 +164,6 @@ def test_update_with_config_dont_update(self): def test_update_with_config_fix_all(self): """Fix the version of all nf-core modules""" - remove_template_modules(self) self.mods_install_trimgalore.install("trimgalore") # Fix the version of all nf-core modules in the .nf-core.yml to an old version @@ -193,7 +187,6 @@ def test_update_with_config_fix_all(self): def test_update_with_config_no_updates(self): """Don't update any nf-core modules""" - remove_template_modules(self) self.mods_install_old.install("trimgalore") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() @@ -227,7 +220,6 @@ def test_update_with_config_no_updates(self): def test_update_different_branch_single_module(self): """Try updating a module in a specific branch""" - remove_template_modules(self) install_obj = ModuleInstall( self.pipeline_dir, prompt=False, @@ -254,7 +246,6 @@ def test_update_different_branch_single_module(self): def test_update_different_branch_mixed_modules_main(self): """Try updating all modules where MultiQC is installed from main branch""" - remove_template_modules(self) # Install fastp assert self.mods_install_gitlab_old.install("fastp") @@ -281,7 +272,6 @@ def test_update_different_branch_mixed_modules_main(self): def test_update_different_branch_mix_modules_branch_test(self): """Try updating all modules where MultiQC is installed from branch-test branch""" - remove_template_modules(self) # Install multiqc from the branch-test branch assert self.mods_install_gitlab_old.install( "multiqc" diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 1c451d8964..6c04c9ad22 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -10,7 +10,6 @@ GITLAB_REPO, GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, - remove_template_modules, with_temporary_folder, ) @@ -63,7 +62,6 @@ def test_subworkflows_install_bam_sort_stats_samtools_twice(self): def test_subworkflows_install_from_gitlab(self): """Test installing a subworkflow from GitLab""" - remove_template_modules(self) assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is True # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) @@ -142,9 +140,3 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self): "installed_by" ] ) == sorted(["subworkflows", "bam_sort_stats_samtools"]) - - -def test_subworkflows_install_alternate_remote(self): - """Test installing a subworkflow from a different remote with the same organization path""" - self.subworkflow_install.install("bam_sort_stats_samtools") - assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is False diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index cde6808bb8..c65999d42c 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -2,7 +2,7 @@ import nf_core.subworkflows -from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, remove_template_modules +from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL def test_subworkflows_list_remote(self): @@ -40,7 +40,6 @@ def test_subworkflows_install_and_list_subworkflows(self): def test_subworkflows_install_gitlab_and_list_subworkflows(self): """Test listing locally installed subworkflows""" - remove_template_modules(self) self.subworkflow_install_gitlab.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() diff --git a/tests/test_modules.py b/tests/test_modules.py index 057e77737c..21c003112e 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -59,10 +59,9 @@ def setUp(self): # Set up the schema root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") - self.pipeline_name = "mypipeline" - self.pipeline_dir = os.path.join(self.tmp_dir, self.pipeline_name) + self.pipeline_dir = os.path.join(self.tmp_dir, "mypipeline") nf_core.create.PipelineCreate( - self.pipeline_name, "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True + "mypipeline", "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True ).init_pipeline() # Set up install objects self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) @@ -151,7 +150,6 @@ def test_modulesrepo_class(self): test_modules_info_remote_gitlab, ) from .modules.install import ( - test_modules_install_alternate_remote, test_modules_install_different_branch_fail, test_modules_install_different_branch_succeed, test_modules_install_emptypipeline, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 27373d2478..19a090f7f0 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -48,10 +48,9 @@ def setUp(self): # Set up the pipeline structure root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") - self.pipeline_name = "mypipeline" - self.pipeline_dir = os.path.join(self.tmp_dir, self.pipeline_name) + self.pipeline_dir = os.path.join(self.tmp_dir, "mypipeline") nf_core.create.PipelineCreate( - self.pipeline_name, "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True + "mypipeline", "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True ).init_pipeline() # Set up the nf-core/modules repo dummy @@ -109,7 +108,6 @@ def tearDown(self): ) from .subworkflows.install import ( test_subworkflow_install_nopipeline, - test_subworkflows_install_alternate_remote, test_subworkflows_install_bam_sort_stats_samtools, test_subworkflows_install_bam_sort_stats_samtools_twice, test_subworkflows_install_different_branch_fail, diff --git a/tests/utils.py b/tests/utils.py index f33af2a7af..0dd60dd051 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,8 +10,6 @@ import responses -import nf_core.modules - OLD_TRIMGALORE_SHA = "06348dffce2a732fc9e656bdc5c64c3e02d302cb" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" @@ -106,19 +104,3 @@ def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module, version): ], } rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) - - -def remove_template_modules(self): - # Remove modules that may cause org_path conflict - workflow_path = Path(self.pipeline_dir, "workflows", self.pipeline_name + ".nf") - with open(workflow_path, "r") as fh: - lines = fh.readlines() - with open(workflow_path, "w") as fh: - for line in lines: - if not line.startswith("include {"): - fh.write(line) - - remove_obj = nf_core.modules.ModuleRemove(self.pipeline_dir, no_pull=False) - - for i in ["multiqc", "fastqc", "custom/dumpsoftwareversions"]: - remove_obj.remove(i)