diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 6ad6b7b157..3ef0760513 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -65,6 +65,12 @@ runs: run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; working-directory: create-lint-wf + # Add empty ro-crate file + - name: add empty ro-crate file + shell: bash + run: touch nf-core-testpipeline/ro-crate-metadata.json + working-directory: create-lint-wf + # Run nf-core pipelines linting - name: nf-core pipelines lint shell: bash diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index b38c1ea5a1..fffa9ffe7a 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -147,6 +147,11 @@ jobs: run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; working-directory: create-test-lint-wf + # Add empty ro-crate file + - name: add empty ro-crate file + run: touch my-prefix-testpipeline/ro-crate-metadata.json + working-directory: create-test-lint-wf + # Run nf-core linting - name: nf-core pipelines lint run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2d67b1ce98..76d5d710c0 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -132,7 +132,7 @@ jobs: - name: Test with pytest run: | - python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 && exit_code=0|| exit_code=$? + python3 -m pytest tests/${{matrix.test}} --color=yes --cov --cov-config=.coveragerc --durations=0 && exit_code=0|| exit_code=$? # don't fail if no tests were collected, e.g. for test_licence.py if [ "${exit_code}" -eq 5 ]; then echo "No tests were collected" diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 81d088e13d..9f16188e95 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -36,6 +36,7 @@ pipelines_launch, pipelines_lint, pipelines_list, + pipelines_rocrate, pipelines_schema_build, pipelines_schema_docs, pipelines_schema_lint, @@ -86,7 +87,7 @@ }, { "name": "For developers", - "commands": ["create", "lint", "bump-version", "sync", "schema", "create-logo"], + "commands": ["create", "lint", "bump-version", "sync", "schema", "rocrate", "create-logo"], }, ], "nf-core modules": [ @@ -570,6 +571,44 @@ def command_pipelines_list(ctx, keywords, sort, json, show_archived): pipelines_list(ctx, keywords, sort, json, show_archived) +# nf-core pipelines rocrate +@pipelines.command("rocrate") +@click.argument( + "pipeline_dir", + type=click.Path(exists=True), + default=Path.cwd(), + required=True, + metavar="", +) +@click.option( + "-j", + "--json_path", + default=Path.cwd(), + type=str, + help="Path to save RO Crate metadata json file to", +) +@click.option("-z", "--zip_path", type=str, help="Path to save RO Crate zip file to") +@click.option( + "-pv", + "--pipeline_version", + type=str, + help="Version of pipeline to use for RO Crate", + default="", +) +@click.pass_context +def rocrate( + ctx, + pipeline_dir: str, + json_path: str, + zip_path: str, + pipeline_version: str, +): + """ + Make an Research Object Crate + """ + pipelines_rocrate(ctx, pipeline_dir, json_path, zip_path, pipeline_version) + + # nf-core pipelines sync @pipelines.command("sync") @click.pass_context @@ -1758,7 +1797,7 @@ def command_schema_validate(pipeline, params): @click.option( "--url", type=str, - default="https://nf-co.re/pipeline_schema_builder", + default="https://oldsite.nf-co.re/pipeline_schema_builder", help="Customise the builder URL (for development work)", ) def command_schema_build(directory, no_prompts, web_only, url): diff --git a/nf_core/commands_pipelines.py b/nf_core/commands_pipelines.py index 1186935e52..3b28f4979c 100644 --- a/nf_core/commands_pipelines.py +++ b/nf_core/commands_pipelines.py @@ -2,6 +2,7 @@ import os import sys from pathlib import Path +from typing import Optional, Union import rich @@ -277,6 +278,33 @@ def pipelines_list(ctx, keywords, sort, json, show_archived): stdout.print(list_workflows(keywords, sort, json, show_archived)) +# nf-core pipelines rocrate +def pipelines_rocrate( + ctx, + pipeline_dir: Union[str, Path], + json_path: Optional[Union[str, Path]], + zip_path: Optional[Union[str, Path]], + pipeline_version: str, +) -> None: + from nf_core.pipelines.rocrate import ROCrate + + if json_path is None and zip_path is None: + log.error("Either `--json_path` or `--zip_path` must be specified.") + sys.exit(1) + else: + pipeline_dir = Path(pipeline_dir) + if json_path is not None: + json_path = Path(json_path) + if zip_path is not None: + zip_path = Path(zip_path) + try: + rocrate_obj = ROCrate(pipeline_dir, pipeline_version) + rocrate_obj.create_rocrate(json_path=json_path, zip_path=zip_path) + except (UserWarning, LookupError, FileNotFoundError) as e: + log.error(e) + sys.exit(1) + + # nf-core pipelines sync def pipelines_sync(ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr): """ diff --git a/nf_core/components/info.py b/nf_core/components/info.py index f3e5bf617c..31769785a1 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -211,9 +211,9 @@ def get_local_yaml(self) -> Optional[Dict]: return yaml.safe_load(fh) log.debug(f"{self.component_type[:-1].title()} '{self.component}' meta.yml not found locally") - return None + return {} - def get_remote_yaml(self) -> Optional[dict]: + def get_remote_yaml(self) -> Optional[Dict]: """Attempt to get the meta.yml file from a remote repo. Returns: diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 9ae735b1c0..15e98ffff8 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1129,8 +1129,10 @@ def dump(self, run_prettier: bool = False) -> None: """ Sort the modules.json, and write it to file """ + # Sort the modules.json + if self.modules_json is None: + self.load() if self.modules_json is not None: - # Sort the modules.json self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) if run_prettier: dump_json_with_prettier(self.modules_json_path, self.modules_json) diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index 8ab547c1cc..dba0a40caf 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -21,6 +21,7 @@ from nf_core.pipelines.create.utils import CreateConfig, features_yml_path, load_features_yaml from nf_core.pipelines.create_logo import create_logo from nf_core.pipelines.lint_utils import run_prettier_on_file +from nf_core.pipelines.rocrate import ROCrate from nf_core.utils import LintConfigType, NFCoreTemplateConfig log = logging.getLogger(__name__) @@ -356,6 +357,11 @@ def render_template(self) -> None: # Make a logo and save it, if it is a nf-core pipeline self.make_pipeline_logo() + if self.config.skip_features is None or "ro-crate" not in self.config.skip_features: + # Create the RO-Crate metadata file + rocrate_obj = ROCrate(self.outdir) + rocrate_obj.create_rocrate(json_path=self.outdir / "ro-crate-metadata.json") + # Update the .nf-core.yml with linting configurations self.fix_linting() diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index 5f23f5e03e..9841879e83 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -432,6 +432,19 @@ seqera_platform: You can extend this file adding any other desired configuration. nfcore_pipelines: False custom_pipelines: True +rocrate: + skippable_paths: + - "ro-crate-metadata.json" + short_description: "Add RO-Crate metadata" + description: "Add a RO-Crate metadata file to describe the pipeline" + help_text: | + RO-Crate is a metadata specification to describe research data and software. + This will add a `ro-crate-metadata.json` file to describe the pipeline. + nfcore_pipelines: False + custom_pipelines: True + linting: + files_warn: + - "ro-crate-metadata.json" vscode: skippable_paths: - ".vscode" diff --git a/nf_core/pipelines/lint/files_exist.py b/nf_core/pipelines/lint/files_exist.py index 9dd307d8b5..19c2498263 100644 --- a/nf_core/pipelines/lint/files_exist.py +++ b/nf_core/pipelines/lint/files_exist.py @@ -66,6 +66,7 @@ def files_exist(self) -> Dict[str, List[str]]: conf/igenomes.config .github/workflows/awstest.yml .github/workflows/awsfulltest.yml + ro-crate-metadata.json Files that *must not* be present, due to being renamed or removed in the template: @@ -171,6 +172,7 @@ def files_exist(self) -> Dict[str, List[str]]: [Path(".github", "workflows", "awstest.yml")], [Path(".github", "workflows", "awsfulltest.yml")], [Path("modules.json")], + [Path("ro-crate-metadata.json")], ] # List of strings. Fails / warns if any of the strings exist. @@ -198,6 +200,12 @@ def files_exist(self) -> Dict[str, List[str]]: ] files_warn_ifexists = [Path(".travis.yml")] + files_hint = [ + [ + ["ro-crate-metadata.json"], + ". Run `nf-core rocrate` to generate this file. Read more about RO-Crates in the [nf-core/tools docs](https://nf-co.re/tools#create-a-ro-crate-metadata-file).", + ], + ] # Remove files that should be ignored according to the linting config ignore_files = self.lint_config.get("files_exist", []) if self.lint_config is not None else [] @@ -225,7 +233,11 @@ def pf(file_path: Union[str, Path]) -> Path: if any([pf(f).is_file() for f in files]): passed.append(f"File found: {self._wrap_quotes(files)}") else: - warned.append(f"File not found: {self._wrap_quotes(files)}") + hint = "" + for file_hint in files_hint: + if file_hint[0] == files: + hint = str(file_hint[1]) + warned.append(f"File not found: {self._wrap_quotes(files)}{hint}") # Files that cause an error if they exist for file in files_fail_ifexists: diff --git a/nf_core/pipelines/rocrate.py b/nf_core/pipelines/rocrate.py new file mode 100644 index 0000000000..915f203f00 --- /dev/null +++ b/nf_core/pipelines/rocrate.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python +"""Code to deal with pipeline RO (Research Object) Crates""" + +import logging +import os +import re +import sys +from datetime import datetime +from pathlib import Path +from typing import Optional, Set, Union + +import requests +import rocrate.rocrate +from git import GitCommandError, InvalidGitRepositoryError +from repo2rocrate.nextflow import NextflowCrateBuilder +from rich.progress import BarColumn, Progress +from rocrate.model.person import Person +from rocrate.rocrate import ROCrate as BaseROCrate + +from nf_core.utils import Pipeline + +log = logging.getLogger(__name__) + + +class CustomNextflowCrateBuilder(NextflowCrateBuilder): + DATA_ENTITIES = NextflowCrateBuilder.DATA_ENTITIES + [ + ("docs/usage.md", "File", "Usage documentation"), + ("docs/output.md", "File", "Output documentation"), + ("suborkflows/local", "Dataset", "Pipeline-specific suborkflows"), + ("suborkflows/nf-core", "Dataset", "nf-core suborkflows"), + (".nf-core.yml", "File", "nf-core configuration file, configuring template features and linting rules"), + (".pre-commit-config.yaml", "File", "Configuration file for pre-commit hooks"), + (".prettierignore", "File", "Ignore file for prettier"), + (".prettierrc", "File", "Configuration file for prettier"), + ] + + +def custom_make_crate( + root: Path, + workflow: Optional[Path] = None, + repo_url: Optional[str] = None, + wf_name: Optional[str] = None, + wf_version: Optional[str] = None, + lang_version: Optional[str] = None, + ci_workflow: Optional[str] = "ci.yml", + diagram: Optional[Path] = None, +) -> BaseROCrate: + builder = CustomNextflowCrateBuilder(root, repo_url=repo_url) + + return builder.build( + workflow, + wf_name=wf_name, + wf_version=wf_version, + lang_version=lang_version, + license=None, + ci_workflow=ci_workflow, + diagram=diagram, + ) + + +class ROCrate: + """ + Class to generate an RO Crate for a pipeline + + """ + + def __init__(self, pipeline_dir: Path, version="") -> None: + """ + Initialise the ROCrate object + + Args: + pipeline_dir (Path): Path to the pipeline directory + version (str): Version of the pipeline to checkout + """ + from nf_core.utils import is_pipeline_directory, setup_requests_cachedir + + is_pipeline_directory(pipeline_dir) + self.pipeline_dir = pipeline_dir + self.version: str = version + self.crate: rocrate.rocrate.ROCrate + self.pipeline_obj = Pipeline(self.pipeline_dir) + self.pipeline_obj._load() + + setup_requests_cachedir() + + def create_rocrate(self, json_path: Union[None, Path] = None, zip_path: Union[None, Path] = None) -> bool: + """ + Create an RO Crate for a pipeline + + Args: + outdir (Path): Path to the output directory + json_path (Path): Path to the metadata file + zip_path (Path): Path to the zip file + + """ + + # Check that the checkout pipeline version is the same as the requested version + if self.version != "": + if self.version != self.pipeline_obj.nf_config.get("manifest.version"): + # using git checkout to get the requested version + log.info(f"Checking out pipeline version {self.version}") + if self.pipeline_obj.repo is None: + log.error(f"Pipeline repository not found in {self.pipeline_dir}") + sys.exit(1) + try: + self.pipeline_obj.repo.git.checkout(self.version) + self.pipeline_obj = Pipeline(self.pipeline_dir) + self.pipeline_obj._load() + except InvalidGitRepositoryError: + log.error(f"Could not find a git repository in {self.pipeline_dir}") + sys.exit(1) + except GitCommandError: + log.error(f"Could not checkout version {self.version}") + sys.exit(1) + self.version = self.pipeline_obj.nf_config.get("manifest.version", "") + self.make_workflow_rocrate() + + # Save just the JSON metadata file + if json_path is not None: + if json_path.name == "ro-crate-metadata.json": + json_path = json_path.parent + + log.info(f"Saving metadata file to '{json_path}'") + self.crate.metadata.write(json_path) + + # Save the whole crate zip file + if zip_path is not None: + if zip_path.name != "ro-crate.crate.zip": + zip_path = zip_path / "ro-crate.crate.zip" + log.info(f"Saving zip file '{zip_path}") + self.crate.write_zip(zip_path) + + if json_path is None and zip_path is None: + log.error("Please provide a path to save the ro-crate file or the zip file.") + return False + + return True + + def make_workflow_rocrate(self) -> None: + """ + Create an RO Crate for a pipeline + """ + if self.pipeline_obj is None: + raise ValueError("Pipeline object not loaded") + + diagram: Optional[Path] = None + # find files (metro|tube)_?(map)?.png in the pipeline directory or docs/ using pathlib + pattern = re.compile(r".*?(metro|tube|subway)_(map).*?\.png", re.IGNORECASE) + for file in self.pipeline_dir.rglob("*.png"): + if pattern.match(file.name): + log.debug(f"Found diagram: {file}") + diagram = file.relative_to(self.pipeline_dir) + break + + # Create the RO Crate object + + self.crate = custom_make_crate( + self.pipeline_dir, + self.pipeline_dir / "main.nf", + self.pipeline_obj.nf_config.get("manifest.homePage", ""), + self.pipeline_obj.nf_config.get("manifest.name", ""), + self.pipeline_obj.nf_config.get("manifest.version", ""), + self.pipeline_obj.nf_config.get("manifest.nextflowVersion", ""), + diagram=diagram, + ) + + # add readme as description + readme = self.pipeline_dir / "README.md" + + try: + self.crate.description = readme.read_text() + except FileNotFoundError: + log.error(f"Could not find README.md in {self.pipeline_dir}") + # get license from LICENSE file + license_file = self.pipeline_dir / "LICENSE" + try: + license = license_file.read_text() + if license.startswith("MIT"): + self.crate.license = "MIT" + else: + # prompt for license + log.info("Could not determine license from LICENSE file") + self.crate.license = input("Please enter the license for this pipeline: ") + except FileNotFoundError: + log.error(f"Could not find LICENSE file in {self.pipeline_dir}") + + self.crate.add_jsonld( + {"@id": "https://nf-co.re/", "@type": "Organization", "name": "nf-core", "url": "https://nf-co.re/"} + ) + + # Set metadata for main entity file + self.set_main_entity("main.nf") + + def set_main_entity(self, main_entity_filename: str): + """ + Set the main.nf as the main entity of the crate and add necessary metadata + """ + if self.crate.mainEntity is None: + raise ValueError("Main entity not set") + + self.crate.mainEntity.append_to( + "dct:conformsTo", "https://bioschemas.org/profiles/ComputationalWorkflow/1.0-RELEASE/", compact=True + ) + # add dateCreated and dateModified, based on the current data + self.crate.mainEntity.append_to("dateCreated", self.crate.root_dataset.get("dateCreated", ""), compact=True) + self.crate.mainEntity.append_to( + "dateModified", str(datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")), compact=True + ) + self.crate.mainEntity.append_to("sdPublisher", {"@id": "https://nf-co.re/"}, compact=True) + if self.version.endswith("dev"): + url = "dev" + else: + url = self.version + self.crate.mainEntity.append_to( + "url", f"https://nf-co.re/{self.crate.name.replace('nf-core/','')}/{url}/", compact=True + ) + self.crate.mainEntity.append_to("version", self.version, compact=True) + + # get keywords from nf-core website + remote_workflows = requests.get("https://nf-co.re/pipelines.json").json()["remote_workflows"] + # go through all remote workflows and find the one that matches the pipeline name + topics = ["nf-core", "nextflow"] + for remote_wf in remote_workflows: + assert self.pipeline_obj.pipeline_name is not None # mypy + if remote_wf["name"] == self.pipeline_obj.pipeline_name.replace("nf-core/", ""): + topics = topics + remote_wf["topics"] + break + + log.debug(f"Adding topics: {topics}") + self.crate.mainEntity.append_to("keywords", topics) + + self.add_main_authors(self.crate.mainEntity) + + self.crate.mainEntity = self.crate.mainEntity + + self.crate.mainEntity.append_to("license", self.crate.license) + self.crate.mainEntity.append_to("name", self.crate.name) + + if "dev" in self.version: + self.crate.creativeWorkStatus = "InProgress" + else: + self.crate.creativeWorkStatus = "Stable" + if self.pipeline_obj.repo is None: + log.error(f"Pipeline repository not found in {self.pipeline_dir}") + else: + tags = self.pipeline_obj.repo.tags + if tags: + # get the tag for this version + for tag in tags: + if tag.commit.hexsha == self.pipeline_obj.repo.head.commit.hexsha: + self.crate.mainEntity.append_to( + "dateCreated", + tag.commit.committed_datetime.strftime("%Y-%m-%dT%H:%M:%SZ"), + compact=True, + ) + + def add_main_authors(self, wf_file: rocrate.model.entity.Entity) -> None: + """ + Add workflow authors to the crate + """ + # add author entity to crate + + try: + authors = self.pipeline_obj.nf_config["manifest.author"].split(",") + # remove spaces + authors = [a.strip() for a in authors] + # add manifest authors as maintainer to crate + + except KeyError: + log.error("No author field found in manifest of nextflow.config") + return + # look at git contributors for author names + try: + git_contributors: Set[str] = set() + if self.pipeline_obj.repo is None: + log.info("No git repository found. No git contributors will be added as authors.") + return + commits_touching_path = list(self.pipeline_obj.repo.iter_commits(paths="main.nf")) + + for commit in commits_touching_path: + if commit.author.name is not None: + git_contributors.add(commit.author.name) + # exclude bots + contributors = {c for c in git_contributors if not c.endswith("bot") and c != "Travis CI User"} + + log.debug(f"Found {len(contributors)} git authors") + + progress_bar = Progress( + "[bold blue]{task.description}", + BarColumn(bar_width=None), + "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", + transient=True, + disable=os.environ.get("HIDE_PROGRESS", None) is not None, + ) + with progress_bar: + bump_progress = progress_bar.add_task( + "Searching for author names on GitHub", total=len(contributors), test_name="" + ) + + for git_author in contributors: + progress_bar.update(bump_progress, advance=1, test_name=git_author) + git_author = ( + requests.get(f"https://api.github.com/users/{git_author}").json().get("name", git_author) + ) + if git_author is None: + log.debug(f"Could not find name for {git_author}") + continue + + except AttributeError: + log.debug("Could not find git contributors") + + # remove usernames (just keep names with spaces) + named_contributors = {c for c in contributors if " " in c} + + for author in named_contributors: + log.debug(f"Adding author: {author}") + + if self.pipeline_obj.repo is None: + log.info("No git repository found. No git contributors will be added as authors.") + return + # get email from git log + email = self.pipeline_obj.repo.git.log(f"--author={author}", "--pretty=format:%ae", "-1") + orcid = get_orcid(author) + author_entitity = self.crate.add( + Person( + self.crate, orcid if orcid is not None else "#" + email, properties={"name": author, "email": email} + ) + ) + wf_file.append_to("creator", author_entitity) + if author in authors: + wf_file.append_to("maintainer", author_entitity) + + +def get_orcid(name: str) -> Optional[str]: + """ + Get the ORCID for a given name + + Args: + name (str): Name of the author + + Returns: + str: ORCID URI or None + """ + base_url = "https://pub.orcid.org/v3.0/search/" + headers = { + "Accept": "application/json", + } + params = {"q": f'family-name:"{name.split()[-1]}" AND given-names:"{name.split()[0]}"'} + response = requests.get(base_url, params=params, headers=headers) + + if response.status_code == 200: + json_response = response.json() + if json_response.get("num-found") == 1: + orcid_uri = json_response.get("result")[0].get("orcid-identifier", {}).get("uri") + log.info(f"Using found ORCID for {name}. Please double-check: {orcid_uri}") + return orcid_uri + else: + log.debug(f"No exact ORCID found for {name}. See {response.url}") + return None + else: + log.info(f"API request to ORCID unsuccessful. Status code: {response.status_code}") + return None diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 2a8a9a1929..61fd6bc2d7 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -43,7 +43,7 @@ def __init__(self): self.schema_from_scratch = False self.no_prompts = False self.web_only = False - self.web_schema_build_url = "https://nf-co.re/pipeline_schema_builder" + self.web_schema_build_url = "https://oldsite.nf-co.re/pipeline_schema_builder" self.web_schema_build_web_url = None self.web_schema_build_api_url = None self.validation_plugin = None @@ -958,6 +958,7 @@ def launch_web_builder(self): """ Send pipeline schema to web builder and wait for response """ + content = { "post_content": "json_schema", "api": "true", @@ -966,6 +967,7 @@ def launch_web_builder(self): "schema": json.dumps(self.schema), } web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_build_url, content) + try: if "api_url" not in web_response: raise AssertionError('"api_url" not in web_response') diff --git a/nf_core/utils.py b/nf_core/utils.py index 16125aed33..9aa0bd589a 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -5,6 +5,7 @@ import concurrent.futures import datetime import errno +import fnmatch import hashlib import io import json @@ -19,7 +20,7 @@ import time from contextlib import contextmanager from pathlib import Path -from typing import Any, Callable, Dict, Generator, List, Literal, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Literal, Optional, Tuple, Union import git import prompt_toolkit.styles @@ -36,6 +37,9 @@ import nf_core +if TYPE_CHECKING: + from nf_core.pipelines.schema import PipelineSchema + log = logging.getLogger(__name__) # ASCII nf-core logo @@ -52,14 +56,29 @@ [ ("qmark", "fg:ansiblue bold"), # token in front of the question ("question", "bold"), # question text - ("answer", "fg:ansigreen nobold bg:"), # submitted answer text behind the question - ("pointer", "fg:ansiyellow bold"), # pointer used in select and checkbox prompts - ("highlighted", "fg:ansiblue bold"), # pointed-at choice in select and checkbox prompts - ("selected", "fg:ansiyellow noreverse bold"), # style for a selected item of a checkbox + ( + "answer", + "fg:ansigreen nobold bg:", + ), # submitted answer text behind the question + ( + "pointer", + "fg:ansiyellow bold", + ), # pointer used in select and checkbox prompts + ( + "highlighted", + "fg:ansiblue bold", + ), # pointed-at choice in select and checkbox prompts + ( + "selected", + "fg:ansiyellow noreverse bold", + ), # style for a selected item of a checkbox ("separator", "fg:ansiblack"), # separator in lists ("instruction", ""), # user instructions for select, rawselect, checkbox ("text", ""), # plain text - ("disabled", "fg:gray italic"), # disabled choices for select and checkbox prompts + ( + "disabled", + "fg:gray italic", + ), # disabled choices for select and checkbox prompts ("choice-default", "fg:ansiblack"), ("choice-default-changed", "fg:ansiyellow"), ("choice-required", "fg:ansired"), @@ -79,7 +98,11 @@ def fetch_remote_version(source_url): return remote_version -def check_if_outdated(current_version=None, remote_version=None, source_url="https://nf-co.re/tools_version"): +def check_if_outdated( + current_version=None, + remote_version=None, + source_url="https://nf-co.re/tools_version", +): """ Check if the current version of nf-core is outdated """ @@ -146,11 +169,12 @@ def __init__(self, wf_path: Path) -> None: self.wf_path = Path(wf_path) self.pipeline_name: Optional[str] = None self.pipeline_prefix: Optional[str] = None - self.schema_obj: Optional[Dict] = None + self.schema_obj: Optional[PipelineSchema] = None + self.repo: Optional[git.Repo] = None try: - repo = git.Repo(self.wf_path) - self.git_sha = repo.head.object.hexsha + self.repo = git.Repo(self.wf_path) + self.git_sha = self.repo.head.object.hexsha except Exception as e: log.debug(f"Could not find git hash for pipeline: {self.wf_path}. {e}") @@ -254,7 +278,7 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: """ log.debug(f"Got '{wf_path}' as path") - + wf_path = Path(wf_path) config = {} cache_fn = None cache_basedir = None @@ -441,6 +465,7 @@ def poll_nfcore_web_api(api_url: str, post_data: Optional[Dict] = None) -> Dict: if post_data is None: response = requests.get(api_url, headers={"Cache-Control": "no-cache"}) else: + log.debug(f"requesting {api_url} with {post_data}") response = requests.post(url=api_url, data=post_data) except requests.exceptions.Timeout: raise AssertionError(f"URL timed out: {api_url}") @@ -526,7 +551,8 @@ def __call__(self, r): with open(gh_cli_config_fn) as fh: gh_cli_config = yaml.safe_load(fh) self.auth = requests.auth.HTTPBasicAuth( - gh_cli_config["github.com"]["user"], gh_cli_config["github.com"]["oauth_token"] + gh_cli_config["github.com"]["user"], + gh_cli_config["github.com"]["oauth_token"], ) self.auth_mode = f"gh CLI config: {gh_cli_config['github.com']['user']}" except Exception: @@ -794,12 +820,18 @@ def get_tag_date(tag_date): # Obtain version and build match = re.search(r"(?::)+([A-Za-z\d\-_.]+)", img["image_name"]) if match is not None: - all_docker[match.group(1)] = {"date": get_tag_date(img["updated"]), "image": img} + all_docker[match.group(1)] = { + "date": get_tag_date(img["updated"]), + "image": img, + } elif img["image_type"] == "Singularity": # Obtain version and build match = re.search(r"(?::)+([A-Za-z\d\-_.]+)", img["image_name"]) if match is not None: - all_singularity[match.group(1)] = {"date": get_tag_date(img["updated"]), "image": img} + all_singularity[match.group(1)] = { + "date": get_tag_date(img["updated"]), + "image": img, + } # Obtain common builds from Docker and Singularity images common_keys = list(all_docker.keys() & all_singularity.keys()) current_date = None @@ -929,13 +961,19 @@ def prompt_pipeline_release_branch( # Releases if len(wf_releases) > 0: for tag in map(lambda release: release.get("tag_name"), wf_releases): - tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] + tag_display = [ + ("fg:ansiblue", f"{tag} "), + ("class:choice-default", "[release]"), + ] choices.append(questionary.Choice(title=tag_display, value=tag)) tag_set.append(str(tag)) # Branches for branch in wf_branches.keys(): - branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] + branch_display = [ + ("fg:ansiyellow", f"{branch} "), + ("class:choice-default", "[branch]"), + ] choices.append(questionary.Choice(title=branch_display, value=branch)) tag_set.append(branch) @@ -966,7 +1004,8 @@ def validate(self, value): return True else: raise questionary.ValidationError( - message="Invalid remote cache index file", cursor_position=len(value.text) + message="Invalid remote cache index file", + cursor_position=len(value.text), ) else: return True @@ -996,7 +1035,13 @@ def get_repo_releases_branches(pipeline, wfs): pipeline = wf.full_name # Store releases and stop loop - wf_releases = list(sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True)) + wf_releases = list( + sorted( + wf.releases, + key=lambda k: k.get("published_at_timestamp", 0), + reverse=True, + ) + ) break # Arbitrary GitHub repo @@ -1016,7 +1061,13 @@ def get_repo_releases_branches(pipeline, wfs): raise AssertionError(f"Not able to find pipeline '{pipeline}'") except AttributeError: # Success! We have a list, which doesn't work with .get() which is looking for a dict key - wf_releases = list(sorted(rel_r.json(), key=lambda k: k.get("published_at_timestamp", 0), reverse=True)) + wf_releases = list( + sorted( + rel_r.json(), + key=lambda k: k.get("published_at_timestamp", 0), + reverse=True, + ) + ) # Get release tag commit hashes if len(wf_releases) > 0: @@ -1224,7 +1275,7 @@ def get_first_available_path(directory: Union[Path, str], paths: List[str]) -> U return None -def sort_dictionary(d): +def sort_dictionary(d: Dict) -> Dict: """Sorts a nested dictionary recursively""" result = {} for k, v in sorted(d.items()): @@ -1365,3 +1416,21 @@ def set_wd(path: Path) -> Generator[None, None, None]: yield finally: os.chdir(start_wd) + + +def get_wf_files(wf_path: Path): + """Return a list of all files in a directory (ignores .gitigore files)""" + + wf_files = [] + + with open(Path(wf_path, ".gitignore")) as f: + lines = f.read().splitlines() + ignore = [line for line in lines if line and not line.startswith("#")] + + for path in Path(wf_path).rglob("*"): + if any(fnmatch.fnmatch(str(path), pattern) for pattern in ignore): + continue + if path.is_file(): + wf_files.append(str(path)) + + return wf_files diff --git a/requirements.txt b/requirements.txt index b7f1c39cab..51259938a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,8 @@ requests requests_cache rich-click==1.8.* rich>=13.3.1 +rocrate +repo2rocrate tabulate textual==0.71.0 trogon diff --git a/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg b/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg index 450f1d303c..c34bd85230 100644 --- a/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg +++ b/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg @@ -19,257 +19,257 @@ font-weight: 700; } - .terminal-333203530-matrix { + .terminal-4061415502-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-333203530-title { + .terminal-4061415502-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-333203530-r1 { fill: #c5c8c6 } -.terminal-333203530-r2 { fill: #e3e3e3 } -.terminal-333203530-r3 { fill: #989898 } -.terminal-333203530-r4 { fill: #e1e1e1 } -.terminal-333203530-r5 { fill: #4ebf71;font-weight: bold } -.terminal-333203530-r6 { fill: #1e1e1e } -.terminal-333203530-r7 { fill: #e2e2e2 } -.terminal-333203530-r8 { fill: #507bb3 } -.terminal-333203530-r9 { fill: #808080 } -.terminal-333203530-r10 { fill: #dde6ed;font-weight: bold } -.terminal-333203530-r11 { fill: #001541 } -.terminal-333203530-r12 { fill: #14191f } -.terminal-333203530-r13 { fill: #0178d4 } -.terminal-333203530-r14 { fill: #454a50 } -.terminal-333203530-r15 { fill: #e2e3e3;font-weight: bold } -.terminal-333203530-r16 { fill: #000000 } -.terminal-333203530-r17 { fill: #e4e4e4 } -.terminal-333203530-r18 { fill: #7ae998 } -.terminal-333203530-r19 { fill: #0a180e;font-weight: bold } -.terminal-333203530-r20 { fill: #008139 } -.terminal-333203530-r21 { fill: #fea62b;font-weight: bold } -.terminal-333203530-r22 { fill: #a7a9ab } -.terminal-333203530-r23 { fill: #e2e3e3 } + .terminal-4061415502-r1 { fill: #c5c8c6 } +.terminal-4061415502-r2 { fill: #e3e3e3 } +.terminal-4061415502-r3 { fill: #989898 } +.terminal-4061415502-r4 { fill: #e1e1e1 } +.terminal-4061415502-r5 { fill: #4ebf71;font-weight: bold } +.terminal-4061415502-r6 { fill: #1e1e1e } +.terminal-4061415502-r7 { fill: #e2e2e2 } +.terminal-4061415502-r8 { fill: #507bb3 } +.terminal-4061415502-r9 { fill: #808080 } +.terminal-4061415502-r10 { fill: #dde6ed;font-weight: bold } +.terminal-4061415502-r11 { fill: #001541 } +.terminal-4061415502-r12 { fill: #14191f } +.terminal-4061415502-r13 { fill: #0178d4 } +.terminal-4061415502-r14 { fill: #454a50 } +.terminal-4061415502-r15 { fill: #e2e3e3;font-weight: bold } +.terminal-4061415502-r16 { fill: #000000 } +.terminal-4061415502-r17 { fill: #e4e4e4 } +.terminal-4061415502-r18 { fill: #7ae998 } +.terminal-4061415502-r19 { fill: #0a180e;font-weight: bold } +.terminal-4061415502-r20 { fill: #008139 } +.terminal-4061415502-r21 { fill: #fea62b;font-weight: bold } +.terminal-4061415502-r22 { fill: #a7a9ab } +.terminal-4061415502-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core pipelines create + nf-core pipelines create - + - - ⭘nf-core pipelines create — Create a new pipeline with the nf-core pipeline templa… - - -Template features - - -▊▔▔▔▔▔▔▔▔▎ -▊▎        Toggle all features -▊▁▁▁▁▁▁▁▁▎ -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Use a GitHub Create a GitHub  Show help  -▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -pipeline. - -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Add Github CI testsThe pipeline will  Show help  -▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -actions for Continuous▃▃ -Integration (CI)  -testing - -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Use reference genomesThe pipeline will be  Hide help  -▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -copy of the most  -common reference  -genome files from  -iGenomes - - -Nf-core pipelines are configured to use a copy of the most common reference  -genome files. - -By selecting this option, your pipeline will include a configuration file  -specifying the paths to these files. - -The required code to use these files will also be included in the template.  -When the pipeline user provides an appropriate genome key, the pipeline will -automatically download the required reference files. -▅▅ -For more information about reference genomes in nf-core pipelines, see the  - - -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Add Github badgesThe README.md file of  Show help  -▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - Back  Continue  -▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - d Toggle dark mode  q Quit  a Toggle all  + + ⭘nf-core pipelines create — Create a new pipeline with the nf-core pipeline templa… + + +Template features + + +▊▔▔▔▔▔▔▔▔▎ +▊▎        Toggle all features +▊▁▁▁▁▁▁▁▁▎ +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use a GitHub Create a GitHub  Show help  +▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +pipeline. + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add Github CI testsThe pipeline will  Show help  +▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +actions for Continuous▅▅ +Integration (CI)  +testing + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use reference genomesThe pipeline will be  Hide help  +▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +copy of the most  +common reference  +genome files from  +iGenomes + + +Nf-core pipelines are configured to use a copy of the most common reference  +genome files. + +By selecting this option, your pipeline will include a configuration file  +specifying the paths to these files. + +The required code to use these files will also be included in the template.  +When the pipeline user provides an appropriate genome key, the pipeline will +automatically download the required reference files. +▅▅ +For more information about reference genomes in nf-core pipelines, see the  + + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add Github badgesThe README.md file of  Show help  +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Continue  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + d Toggle dark mode  q Quit  a Toggle all  diff --git a/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg b/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg index 6e178ba840..b8dea05604 100644 --- a/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg +++ b/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg @@ -19,256 +19,256 @@ font-weight: 700; } - .terminal-3425198753-matrix { + .terminal-1727160999-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3425198753-title { + .terminal-1727160999-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3425198753-r1 { fill: #c5c8c6 } -.terminal-3425198753-r2 { fill: #e3e3e3 } -.terminal-3425198753-r3 { fill: #989898 } -.terminal-3425198753-r4 { fill: #e1e1e1 } -.terminal-3425198753-r5 { fill: #4ebf71;font-weight: bold } -.terminal-3425198753-r6 { fill: #1e1e1e } -.terminal-3425198753-r7 { fill: #0178d4 } -.terminal-3425198753-r8 { fill: #e2e2e2 } -.terminal-3425198753-r9 { fill: #507bb3 } -.terminal-3425198753-r10 { fill: #808080 } -.terminal-3425198753-r11 { fill: #dde6ed;font-weight: bold } -.terminal-3425198753-r12 { fill: #001541 } -.terminal-3425198753-r13 { fill: #14191f } -.terminal-3425198753-r14 { fill: #454a50 } -.terminal-3425198753-r15 { fill: #7ae998 } -.terminal-3425198753-r16 { fill: #e2e3e3;font-weight: bold } -.terminal-3425198753-r17 { fill: #0a180e;font-weight: bold } -.terminal-3425198753-r18 { fill: #000000 } -.terminal-3425198753-r19 { fill: #008139 } -.terminal-3425198753-r20 { fill: #fea62b;font-weight: bold } -.terminal-3425198753-r21 { fill: #a7a9ab } -.terminal-3425198753-r22 { fill: #e2e3e3 } + .terminal-1727160999-r1 { fill: #c5c8c6 } +.terminal-1727160999-r2 { fill: #e3e3e3 } +.terminal-1727160999-r3 { fill: #989898 } +.terminal-1727160999-r4 { fill: #e1e1e1 } +.terminal-1727160999-r5 { fill: #4ebf71;font-weight: bold } +.terminal-1727160999-r6 { fill: #1e1e1e } +.terminal-1727160999-r7 { fill: #0178d4 } +.terminal-1727160999-r8 { fill: #e2e2e2 } +.terminal-1727160999-r9 { fill: #507bb3 } +.terminal-1727160999-r10 { fill: #808080 } +.terminal-1727160999-r11 { fill: #dde6ed;font-weight: bold } +.terminal-1727160999-r12 { fill: #001541 } +.terminal-1727160999-r13 { fill: #14191f } +.terminal-1727160999-r14 { fill: #454a50 } +.terminal-1727160999-r15 { fill: #7ae998 } +.terminal-1727160999-r16 { fill: #e2e3e3;font-weight: bold } +.terminal-1727160999-r17 { fill: #0a180e;font-weight: bold } +.terminal-1727160999-r18 { fill: #000000 } +.terminal-1727160999-r19 { fill: #008139 } +.terminal-1727160999-r20 { fill: #fea62b;font-weight: bold } +.terminal-1727160999-r21 { fill: #a7a9ab } +.terminal-1727160999-r22 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core pipelines create + nf-core pipelines create - + - - ⭘nf-core pipelines create — Create a new pipeline with the nf-core pipeline templa… - - -Template features - - -▊▔▔▔▔▔▔▔▔▎ -▊▎        Toggle all features -▊▁▁▁▁▁▁▁▁▎ -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Use a GitHub Create a GitHub  Show help  -▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -pipeline. - -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Add Github CI testsThe pipeline will  Show help  -▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -actions for Continuous -Integration (CI) ▄▄ -testing - -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Use reference genomesThe pipeline will be  Show help  -▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -copy of the most  -common reference  -genome files from  -iGenomes - -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Add Github badgesThe README.md file of  Show help  -▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -include GitHub badges - -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Add configuration The pipeline will  Show help  -▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -profiles containing  -custom parameters  -required to run  -nf-core pipelines at  -different institutions - -▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -▊▎        Use code lintersThe pipeline will  Show help  -▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - Back  Continue  -▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - d Toggle dark mode  q Quit  a Toggle all  + + ⭘nf-core pipelines create — Create a new pipeline with the nf-core pipeline templa… + + +Template features + + +▊▔▔▔▔▔▔▔▔▎ +▊▎        Toggle all features +▊▁▁▁▁▁▁▁▁▎ +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use a GitHub Create a GitHub  Show help  +▊▁▁▁▁▁▁▁▁▎        repository.repository for the ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +pipeline. + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add Github CI testsThe pipeline will  Show help  +▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +actions for Continuous +Integration (CI) ▇▇ +testing + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use reference genomesThe pipeline will be  Show help  +▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +copy of the most  +common reference  +genome files from  +iGenomes + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add Github badgesThe README.md file of  Show help  +▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +include GitHub badges + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Add configuration The pipeline will  Show help  +▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +profiles containing  +custom parameters  +required to run  +nf-core pipelines at  +different institutions + +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎        Use code lintersThe pipeline will  Show help  +▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + Back  Continue  +▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + d Toggle dark mode  q Quit  a Toggle all  diff --git a/tests/pipelines/test_rocrate.py b/tests/pipelines/test_rocrate.py new file mode 100644 index 0000000000..01a77ecd76 --- /dev/null +++ b/tests/pipelines/test_rocrate.py @@ -0,0 +1,127 @@ +"""Test the nf-core pipelines rocrate command""" + +import shutil +import tempfile +from pathlib import Path + +import git +import rocrate.rocrate +from git import Repo + +import nf_core.pipelines.create +import nf_core.pipelines.create.create +import nf_core.pipelines.rocrate +import nf_core.utils + +from ..test_pipelines import TestPipelines + + +class TestROCrate(TestPipelines): + """Class for lint tests""" + + def setUp(self) -> None: + super().setUp() + # add fake metro map + Path(self.pipeline_dir, "docs", "images", "nf-core-testpipeline_metro_map.png").touch() + # commit the changes + repo = Repo(self.pipeline_dir) + repo.git.add(A=True) + repo.index.commit("Initial commit") + self.rocrate_obj = nf_core.pipelines.rocrate.ROCrate(self.pipeline_dir) + + def tearDown(self): + """Clean up temporary files and folders""" + + if self.tmp_dir.exists(): + shutil.rmtree(self.tmp_dir) + + def test_rocrate_creation(self): + """Run the nf-core rocrate command""" + + # Run the command + self.rocrate_obj + assert self.rocrate_obj.create_rocrate(self.pipeline_dir, self.pipeline_dir) + + # Check that the crate was created + self.assertTrue(Path(self.pipeline_dir, "ro-crate-metadata.json").exists()) + + # Check that the entries in the crate are correct + crate = rocrate.rocrate.ROCrate(self.pipeline_dir) + entities = crate.get_entities() + + # Check if the correct entities are set: + for entity in entities: + entity_json = entity.as_jsonld() + if entity_json["@id"] == "./": + self.assertEqual(entity_json.get("name"), "nf-core/testpipeline") + self.assertEqual(entity_json["mainEntity"], {"@id": "main.nf"}) + elif entity_json["@id"] == "#main.nf": + self.assertEqual(entity_json["programmingLanguage"], [{"@id": "#nextflow"}]) + self.assertEqual(entity_json["image"], [{"@id": "nf-core-testpipeline_metro_map.png"}]) + # assert there is a metro map + # elif entity_json["@id"] == "nf-core-testpipeline_metro_map.png": # FIXME waiting for https://github.com/ResearchObject/ro-crate-py/issues/174 + # self.assertEqual(entity_json["@type"], ["File", "ImageObject"]) + # assert that author is set as a person + elif "name" in entity_json and entity_json["name"] == "Test McTestFace": + self.assertEqual(entity_json["@type"], "Person") + # check that it is set as author of the main entity + if crate.mainEntity is not None: + self.assertEqual(crate.mainEntity["author"][0].id, entity_json["@id"]) + + def test_rocrate_creation_wrong_pipeline_dir(self): + """Run the nf-core rocrate command with a wrong pipeline directory""" + # Run the command + + # Check that it raises a UserWarning + with self.assertRaises(UserWarning): + nf_core.pipelines.rocrate.ROCrate(self.pipeline_dir / "bad_dir") + + # assert that the crate was not created + self.assertFalse(Path(self.pipeline_dir / "bad_dir", "ro-crate-metadata.json").exists()) + + def test_rocrate_creation_with_wrong_version(self): + """Run the nf-core rocrate command with a pipeline version""" + # Run the command + + self.rocrate_obj = nf_core.pipelines.rocrate.ROCrate(self.pipeline_dir, version="1.0.0") + + # Check that the crate was created + with self.assertRaises(SystemExit): + assert self.rocrate_obj.create_rocrate(self.pipeline_dir, self.pipeline_dir) + + def test_rocrate_creation_without_git(self): + """Run the nf-core rocrate command with a pipeline version""" + + self.rocrate_obj = nf_core.pipelines.rocrate.ROCrate(self.pipeline_dir, version="1.0.0") + # remove git repo + shutil.rmtree(self.pipeline_dir / ".git") + # Check that the crate was created + with self.assertRaises(SystemExit): + assert self.rocrate_obj.create_rocrate(self.pipeline_dir, self.pipeline_dir) + + def test_rocrate_creation_to_zip(self): + """Run the nf-core rocrate command with a zip output""" + assert self.rocrate_obj.create_rocrate(self.pipeline_dir, zip_path=self.pipeline_dir) + # Check that the crate was created + self.assertTrue(Path(self.pipeline_dir, "ro-crate.crate.zip").exists()) + + def test_rocrate_creation_for_fetchngs(self): + """Run the nf-core rocrate command with nf-core/fetchngs""" + tmp_dir = Path(tempfile.mkdtemp()) + # git clone nf-core/fetchngs + git.Repo.clone_from("https://github.com/nf-core/fetchngs", tmp_dir / "fetchngs") + # Run the command + self.rocrate_obj = nf_core.pipelines.rocrate.ROCrate(tmp_dir / "fetchngs", version="1.12.0") + assert self.rocrate_obj.create_rocrate(tmp_dir / "fetchngs", self.pipeline_dir) + + # Check that Sateesh Peri is mentioned in creator field + + crate = rocrate.rocrate.ROCrate(self.pipeline_dir) + entities = crate.get_entities() + for entity in entities: + entity_json = entity.as_jsonld() + if entity_json["@id"] == "#main.nf": + assert "https://orcid.org/0000-0002-9879-9070" in entity_json["creator"] + + # Clean up + shutil.rmtree(tmp_dir)