From 140656138757da1536471900a101417c55e4be2c Mon Sep 17 00:00:00 2001 From: vsoch Date: Sat, 19 Mar 2022 20:59:36 -0600 Subject: [PATCH] opening pr to show diff of #511 Signed-off-by: vsoch --- CHANGELOG.md | 3 + docs/conf.py | 2 +- docs/getting_started/user-guide.rst | 73 ++++++++++- shpc/client/__init__.py | 19 ++- shpc/client/add.py | 3 + shpc/client/check.py | 3 + shpc/client/config.py | 23 +--- shpc/client/docgen.py | 2 + shpc/client/get.py | 2 + shpc/client/inspect.py | 3 + shpc/client/install.py | 7 +- shpc/client/listing.py | 2 + shpc/client/namespace.py | 3 + shpc/client/shell.py | 6 +- shpc/client/show.py | 3 + shpc/client/test.py | 2 + shpc/client/uninstall.py | 3 + shpc/main/modules/__init__.py | 118 ++++++++++++++++-- .../modules/templates/default_version.lua | 0 shpc/main/modules/templates/docker.lua | 4 +- shpc/main/modules/templates/docker.tcl | 4 +- .../modules/templates/no_default_version.tcl | 2 + shpc/main/modules/templates/singularity.lua | 4 +- shpc/main/modules/templates/singularity.tcl | 4 +- shpc/main/schemas.py | 2 + shpc/main/settings.py | 44 +++++++ shpc/main/wrappers/base.py | 2 +- shpc/settings.yml | 9 +- shpc/tests/test_container.py | 2 - shpc/tests/test_container_config.py | 2 - shpc/tests/test_settings.py | 1 - shpc/version.py | 2 +- 32 files changed, 312 insertions(+), 47 deletions(-) create mode 100644 shpc/main/modules/templates/default_version.lua create mode 100644 shpc/main/modules/templates/no_default_version.tcl diff --git a/CHANGELOG.md b/CHANGELOG.md index 64878ccad..5693bc7af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pip. Only major versions will be released as tags on Github. ## [0.0.x](https://github.com/singularityhub/singularity-hpc/tree/main) (0.0.x) + - support for installing to symlink tree (0.0.49) + - also including cleanup of symlink tree on uninstall + - ability to set custom config variable on the fly with -c - Properly cleanup empty module directories, and asking to remove a container that doesn't exist now logs a _warning_ (0.0.48) - wrapper script generation permissions error (0.0.47) - fixing but with stream command repeating output (0.0.46) diff --git a/docs/conf.py b/docs/conf.py index e69535a71..bcaf482f9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -290,4 +290,4 @@ def setup(app): - app.add_stylesheet("sphinx-argparse.css") + app.add_css_file("sphinx-argparse.css") diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index 8086122b1..1d45d3eef 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -52,6 +52,11 @@ You can then easily install, load, and use modules: $ module load biocontainers/samtools $ samtools +Or set a configuration value on the fly for any command: + +.. code-block:: console + + $ shpc install -c set:symlink_base:/tmp/modules biocontainers/samtools The above assumes that you've installed the software, and have already added the modules folder to be seen by your module software. If your module @@ -174,11 +179,17 @@ variable replacement. A summary table of variables is included below, and then f * - container_tech - The container technology to use (singularity or podman) - singularity + * - symlink_base + - If set, where you want to install a simplified module tree to using ``--symlink-tree`` + - $root_dir/symlinks + * - symlink_tree + - If set to true, ALWAYS generate a symlink tree given that a symlink base is defined regardless of ``--symlink-tree`` flag + - false * - updated_at - a timestamp to keep track of when you last saved - never * - default_version - - A boolean to indicate generating a .version file (LMOD or lua modules only) + - A boolean to indicate whether a default version will be arbitrarily chosen, when multiple versions are available, and none is explicitly requested - true * - singularity_module - if defined, add to module script to load this Singularity module first @@ -233,6 +244,14 @@ variable replacement. A summary table of variables is included below, and then f - All features default to null +Note that any configuration value can be set permanently by using ``shpc config`` +or manually editing the file, but you can also set config values "one off" as follows: + +.. code-block:: console + + $ shpc install -c set:symlink_base:/tmp/modules ghcr.io/autamus/clingo + + These settings will be discussed in more detail in the following sections. Features @@ -359,6 +378,58 @@ you can add or remove entries via the config variable ``registry`` # Note that "add" is used for lists of things (e.g., the registry config variable is a list) and "set" is used to set a key value pair. +Symlink Base +------------ + +By default, your modules are installed to your ``module_base`` described above with a complete +namespace, meaning the container registry from where they arise. We do this so that the namespace +is consistent and there are no conflicts. However, if you want a simplified tree to install from, +meaning the module full names are _just_ the final container name, you can set the ``symlink_base`` +in your settings to a different root. For example, let's say we want to install a set of modules. +We can use the default ``symlink_base`` of ``$root_dir/symlinks`` or set our own ``symlink_base`` +in the settings.yaml. We could do: + +.. code-block:: console + + $ shpc install ghcr.io/autamus/clingo --symlink-tree + $ shpc install ghcr.io/autamus/samtools --symlink-tree + +Then, for example, if you want to load the modules, you'll see the shorter names are +available! + +.. code-block:: console + + $ module use ./symlinks + $ module load clingo/5.5.1/module + +This is much more efficient compared to the install that uses the full paths: + +.. code-block:: console + + $ module use ./modules + $ module load ghcr.io/autamus/clingo/5.5.1/module + +Since we install based on the container name *and* version tag, this even gives you +the ability to install versions from different container bases in the same root. +If there is a conflict, you will be given the option to exit (and abort) or continue. +Finally, if you need an easy way to run through the containers you've already installed +to create the links: + + +.. code-block:: console + + for module in $(shpc list); do + shpc install $module --symlink-tree + done + +And that will reinstall the modules you have installed, but in their symlink tree location. + + +.. warning:: + + Be cautious about creating symlinks in containers or other contexts where a bind + could eliminate the symlink or make the path non-existent. + Module Names ------------ diff --git a/shpc/client/__init__.py b/shpc/client/__init__.py index f49a6e031..39dcaa02c 100644 --- a/shpc/client/__init__.py +++ b/shpc/client/__init__.py @@ -40,6 +40,17 @@ def get_parser(): help="custom path to settings file.", ) + # On the fly updates to config params + parser.add_argument( + "-c", + dest="config_params", + help=""""customize a config value on the fly to ADD/SET/REMOVE for a command +shpc -c set:key:value +shpc -c add:registry:/tmp/registry +shpc -c rm:registry:/tmp/registry""", + action="append", + ) + parser.add_argument( "--version", dest="version", @@ -89,6 +100,13 @@ def get_parser(): "install_recipe", help="recipe to install\nshpc install python\nshpc install python:3.9.5-alpine", ) + install.add_argument( + "--symlink-tree", + dest="symlink", + help="install to symlink tree too.", + default=False, + action="store_true", + ) # List installed modules listing = subparsers.add_parser("list", description="list installed modules.") @@ -147,7 +165,6 @@ def get_parser(): config.add_argument( "--central", - "-c", dest="central", help="make edits to the central config file.", default=False, diff --git a/shpc/client/add.py b/shpc/client/add.py index 095747b69..d75814920 100644 --- a/shpc/client/add.py +++ b/shpc/client/add.py @@ -13,4 +13,7 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.add(args.sif_path[0], args.module_id[0]) diff --git a/shpc/client/check.py b/shpc/client/check.py index 07b76fad7..1998c0c87 100644 --- a/shpc/client/check.py +++ b/shpc/client/check.py @@ -13,4 +13,7 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.check(args.module_name) diff --git a/shpc/client/config.py b/shpc/client/config.py index 86dc97fe8..ad224a131 100644 --- a/shpc/client/config.py +++ b/shpc/client/config.py @@ -33,25 +33,12 @@ def main(args, parser, extra, subparser): return cli.settings.inituser() if command == "edit": return cli.settings.edit() - elif command in ["set", "add", "remove"]: - for param in args.params: - if ":" not in param: - logger.warning( - "Param %s is missing a :, should be key:value pair. Skipping." - % param - ) - continue - key, value = param.split(":", 1) - if command == "set": - cli.settings.set(key, value) - logger.info("Updated %s to be %s" % (key, value)) - elif command == "add": - cli.settings.add(key, value) - logger.info("Added %s to %s" % (key, value)) - elif command == "remove": - cli.settings.remove(key, value) - logger.info("Removed %s from %s" % (key, value)) + if command in ["set", "add", "remove"]: + + # Update each param + for param in args.params: + cli.settings.update_param(command, param) # Save settings cli.settings.save() diff --git a/shpc/client/docgen.py b/shpc/client/docgen.py index 10e54a4bf..7c6ecb994 100644 --- a/shpc/client/docgen.py +++ b/shpc/client/docgen.py @@ -13,4 +13,6 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.docgen(args.module_name) diff --git a/shpc/client/get.py b/shpc/client/get.py index 47cfde176..a600aec87 100644 --- a/shpc/client/get.py +++ b/shpc/client/get.py @@ -14,6 +14,8 @@ def main(args, parser, extra, subparser): container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) result = cli.get(args.module_name, args.env_file) if result: print(result) diff --git a/shpc/client/inspect.py b/shpc/client/inspect.py index 372c84cec..15999d539 100644 --- a/shpc/client/inspect.py +++ b/shpc/client/inspect.py @@ -15,6 +15,9 @@ def main(args, parser, extra, subparser): settings_file=args.settings_file, container_tech=args.container_tech, ) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) metadata = cli.inspect(args.module_name) # Case 1: dump entire thing as json diff --git a/shpc/client/install.py b/shpc/client/install.py index 2b7cefc4d..839f52de8 100644 --- a/shpc/client/install.py +++ b/shpc/client/install.py @@ -16,4 +16,9 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) - cli.install(args.install_recipe) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) + + # And do the install + cli.install(args.install_recipe, symlink=args.symlink) diff --git a/shpc/client/listing.py b/shpc/client/listing.py index c00fd5358..29d6e097c 100644 --- a/shpc/client/listing.py +++ b/shpc/client/listing.py @@ -14,4 +14,6 @@ def main(args, parser, extra, subparser): container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.list(args.pattern, args.names_only, short=args.short) diff --git a/shpc/client/namespace.py b/shpc/client/namespace.py index 79db0867a..aab992a39 100644 --- a/shpc/client/namespace.py +++ b/shpc/client/namespace.py @@ -13,6 +13,9 @@ def main(args, parser, extra, subparser): cli = get_client(quiet=args.quiet, settings_file=args.settings_file) + # Update config settings on the fly + cli.settings.update_params(args.config_params) + # Case 1: we need to unset a namespace if not args.namespace: sys.exit("Please choose: shpc use or shpc unset.") diff --git a/shpc/client/shell.py b/shpc/client/shell.py index 2fcb3182c..b40f36581 100644 --- a/shpc/client/shell.py +++ b/shpc/client/shell.py @@ -34,13 +34,17 @@ def main(args, parser, extra, subparser): def create_client(args): from shpc.main import get_client - return get_client( + cli = get_client( quiet=args.quiet, settings_file=args.settings_file, module=args.module, container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) + return cli + def ipython(args): """ diff --git a/shpc/client/show.py b/shpc/client/show.py index 716168206..23a7baa70 100644 --- a/shpc/client/show.py +++ b/shpc/client/show.py @@ -8,4 +8,7 @@ def main(args, parser, extra, subparser): from shpc.main import get_client cli = get_client(quiet=args.quiet, settings_file=args.settings_file) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.show(args.name, names_only=not args.versions, filter_string=args.filter_string) diff --git a/shpc/client/test.py b/shpc/client/test.py index 26dca8d82..31d9402b7 100644 --- a/shpc/client/test.py +++ b/shpc/client/test.py @@ -14,6 +14,8 @@ def main(args, parser, extra, subparser): container_tech=args.container_tech, ) + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.test( args.module_name, test_exec=args.test_exec, diff --git a/shpc/client/uninstall.py b/shpc/client/uninstall.py index b599cef19..be40d1987 100644 --- a/shpc/client/uninstall.py +++ b/shpc/client/uninstall.py @@ -13,4 +13,7 @@ def main(args, parser, extra, subparser): module=args.module, container_tech=args.container_tech, ) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) cli.uninstall(args.uninstall_recipe, args.force) diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index 878b7d6ce..b0581afc7 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -11,7 +11,6 @@ from datetime import datetime import os -from pathlib import Path import shutil import subprocess import sys @@ -53,6 +52,8 @@ def _cleanup(self, module_dir): # Exit early if the module directory for some reason was removed if not os.path.exists(module_dir): return + + # Cleanup symlink, then module directory shutil.rmtree(module_dir) # If directories above it are empty, remove @@ -102,13 +103,14 @@ def uninstall(self, name, force=False): def _uninstall(self, module_dir, name, force=False): """ - Sub function, so we can pass more than one folder from uninstall + Uninstall a module directory by name, including possible symlinks. """ if os.path.exists(module_dir): if not force: msg = "%s, and all content below it? " % name if not utils.confirm_uninstall(msg, force): return + self._cleanup_symlink(module_dir) self._cleanup(module_dir) logger.info("%s and all subdirectories have been removed." % name) else: @@ -168,6 +170,81 @@ def list(self, pattern=None, names_only=False, out=None, short=False): short=short, ) + # Symbolik links + + def get_symlink_path(self, module_dir): + """ + Should only be called given a self.settings.symlink_base is set + """ + if not self.settings.symlink_base: + return + + symlink_base_name = os.path.join(self.settings.symlink_base, *module_dir.split(os.sep)[-2:]) + + # With Lmod and default_version==True, the symlinks points to module.lua itself, + # and its name needs to end with `.lua` too + if self.module_extension == "lua" and self.settings.default_version == True: + return symlink_base_name + ".lua" + else: + return symlink_base_name + + def create_symlink(self, module_dir): + """ + Create the symlink if desired by the user! + """ + symlink_path = self.get_symlink_path(module_dir) + if os.path.exists(symlink_path): + os.unlink(symlink_path) + symlink_dir = os.path.dirname(symlink_path) + + # If the parent directory doesn't exist, make it + if not os.path.exists(symlink_dir): + utils.mkdirp([symlink_dir]) + + # With Lmod, default_version==False can't be made to work with symlinks at the module.lua level + if self.module_extension == "lua" and self.settings.default_version == False: + symlink_target = module_dir + else: + symlink_target = os.path.join(module_dir, self.modulefile) + logger.info("Creating link %s -> %s" % (symlink_target, symlink_path)) + + # Create the symbolic link! + os.symlink(symlink_target, symlink_path) + + # Create .version + self.write_version_file(os.path.dirname(symlink_path)) + + def check_symlink(self, module_dir): + """ + Given an install command, if --symlink-tree is provided make + sure we don't already have this symlink in the tree. + """ + # Get the symlink path - does it exist? + symlink_path = self.get_symlink_path(module_dir) + if os.path.exists(symlink_path) and not utils.confirm_action('%s already exists, are you sure you want to overwrite?' % symlink_path): + sys.exit(0) + + + def _cleanup_symlink(self, module_dir): + """ + Remove symlink directories if they exist + """ + symlinked_module = self.get_symlink_path(module_dir) + if not symlinked_module: + return + if os.path.exists(symlinked_module) and os.path.islink(symlinked_module): + os.unlink(symlinked_module) + + # Clean up directories that become empty + parent_dir = os.path.dirname(symlinked_module) + if not os.path.exists(parent_dir): + return + + # If the parent of the symlink only has zero files OR one file .version, cleanup + files = os.listdir(parent_dir) + if len(files) == 0 or (len(files) == 1 and files[0] == ".version"): + shutil.rmtree(parent_dir) + def docgen(self, module_name, out=None): """ Render documentation for a module. @@ -271,7 +348,24 @@ def check(self, module_name): config = self._load_container(module_name.rsplit(":", 1)[0]) return self.container.check(module_name, config) - def install(self, name, tag=None, **kwargs): + def write_version_file(self, version_dir): + """ + Create the .version file, if there is a template for it. + + Note that we don't actually change the content of the template: + it is copied as is. + """ + version_template = 'default_version.' + self.module_extension + if not self.settings.default_version: + version_template = 'no_' + version_template + template_file = os.path.join(here, "templates", version_template) + if os.path.exists(template_file): + version_file = os.path.join(version_dir, ".version") + if not os.path.exists(version_file): + version_content = shpc.utils.read_file(template_file) + shpc.utils.write_file(version_file, version_content) + + def install(self, name, tag=None, symlink=False, **kwargs): """ Given a unique resource identifier, install a recipe. @@ -299,14 +393,18 @@ def install(self, name, tag=None, **kwargs): module_dir = os.path.join(self.settings.module_base, uri, tag.name) subfolder = os.path.join(uri, tag.name) container_dir = self.container.container_dir(subfolder) + + # Global override to arg + symlink = self.settings.symlink_tree is True or symlink + + if symlink: + # Cut out early if symlink desired and already exists + self.check_symlink(module_dir) shpc.utils.mkdirp([module_dir, container_dir]) # Add a .version file to indicate the level of versioning (not for tcl) - if self.module_extension != "tcl" and self.settings.default_version == True: - version_dir = os.path.join(self.settings.module_base, uri) - version_file = os.path.join(version_dir, ".version") - if not os.path.exists(version_file): - Path(version_file).touch() + version_dir = os.path.join(self.settings.module_base, uri) + self.write_version_file(version_dir) # For Singularity this is a path, podman is a uri. If None is returned # there was an error and we cleanup @@ -356,4 +454,8 @@ def install(self, name, tag=None, **kwargs): if ":" not in name: name = "%s:%s" %(name, tag.name) logger.info("Module %s was created." % name) + + if symlink: + self.create_symlink(module_dir) + return container_path diff --git a/shpc/main/modules/templates/default_version.lua b/shpc/main/modules/templates/default_version.lua new file mode 100644 index 000000000..e69de29bb diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index 780311678..a4b6b50a8 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -40,8 +40,8 @@ For each of the above, you can export: if not os.getenv("PODMAN_OPTS") then setenv ("PODMAN_OPTS", "") end if not os.getenv("PODMAN_COMMAND_OPTS") then setenv ("PODMAN_COMMAND_OPTS", "") end --- directory containing this modulefile (dynamically defined) -local moduleDir = myFileName():match("(.*[/])") or "." +-- directory containing this modulefile, once symlinks resolved (dynamically defined) +local moduleDir = subprocess("realpath " .. myFileName()):match("(.*[/])") or "." -- interactive shell to any container, plus exec for aliases local containerPath = '{{ image }}' diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index ade20ceb4..93a25781c 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -55,8 +55,8 @@ set helpcommand "This module is a {{ docker }} container wrapper for {{ name }} {% if labels %}{% for key, value in labels.items() %}set {{ key }} "{{ value }}" {% endfor %}{% endif %} -# directory containing this modulefile (dynamically defined) -set moduleDir "[file dirname ${ModulesCurrentModulefile}]" +# directory containing this modulefile, once symlinks resolved (dynamically defined) +set moduleDir [file dirname [expr { [string equal [file type ${ModulesCurrentModulefile}] "link"] ? [file readlink ${ModulesCurrentModulefile}] : ${ModulesCurrentModulefile} }]] # conflict with modules with the same alias name conflict {{ parsed_name.tool }} diff --git a/shpc/main/modules/templates/no_default_version.tcl b/shpc/main/modules/templates/no_default_version.tcl new file mode 100644 index 000000000..1c52dfc7c --- /dev/null +++ b/shpc/main/modules/templates/no_default_version.tcl @@ -0,0 +1,2 @@ +#%Module +set ModulesVersion "please_specify_a_version_number" diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index f50c7ec4f..84bbe1f6c 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -38,8 +38,8 @@ For each of the above, you can export: {% if settings.singularity_module %}load("{{ settings.singularity_module }}"){% endif %} --- directory containing this modulefile (dynamically defined) -local moduleDir = myFileName():match("(.*[/])") or "." +-- directory containing this modulefile, once symlinks resolved (dynamically defined) +local moduleDir = subprocess("realpath " .. myFileName()):match("(.*[/])") or "." -- singularity environment variable to set shell setenv("SINGULARITY_SHELL", "{{ settings.singularity_shell }}") diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index dad790d99..b2e078dd5 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -59,8 +59,8 @@ set helpcommand "This module is a singularity container wrapper for {{ name }} v {% if labels %}{% for key, value in labels.items() %}set {{ key }} "{{ value }}" {% endfor %}{% endif %} -# directory containing this modulefile (dynamically defined) -set moduleDir "[file dirname ${ModulesCurrentModulefile}]" +# directory containing this modulefile, once symlinks resolved (dynamically defined) +set moduleDir [file dirname [expr { [string equal [file type ${ModulesCurrentModulefile}] "link"] ? [file readlink ${ModulesCurrentModulefile}] : ${ModulesCurrentModulefile} }]] # conflict with modules with the same alias name conflict {{ parsed_name.tool }} diff --git a/shpc/main/schemas.py b/shpc/main/schemas.py index a2beaf393..9d85efc4c 100644 --- a/shpc/main/schemas.py +++ b/shpc/main/schemas.py @@ -139,6 +139,8 @@ "environment_file": {"type": "string"}, "default_version": {"type": "boolean"}, "enable_tty": {"type": "boolean"}, + "symlink_base": {"type": ["string", "null"]}, + "symlink_tree": {"type": "boolean"}, "wrapper_scripts": wrapper_scripts, "container_tech": {"type": "string", "enum": ["singularity", "podman", "docker"]}, "singularity_shell": {"type": "string", "enum": shells}, diff --git a/shpc/main/settings.py b/shpc/main/settings.py index 73cf3cadf..430260881 100644 --- a/shpc/main/settings.py +++ b/shpc/main/settings.py @@ -19,6 +19,7 @@ from datetime import datetime import jsonschema import os +import re def OrderedList(*l): @@ -276,6 +277,49 @@ def __iter__(self): for key, value in self.__dict__.items(): yield key, value + def update_params(self, params): + """ + Update a configuration on the fly (no save) only for set/add/remove. + Unlike the traditional set/get/add functions, this function expects + each entry in the params list to start with the action, e.g.: + + set:name:value + add:name:value + rm:name:value + """ + # Cut out early if params not provided + if not params: + return + + for param in params: + if not re.search("^(add|set|rm)", param, re.IGNORECASE) or ":" not in param: + logger.warning( + "Parameter update request must start with (add|set|rm):, skipping %s" + ) + command, param = param.split(":", 1) + self.update_param(command.lower(), param) + + def update_param(self, command, param): + """ + Given a parameter, update the configuration on the fly if it's in set/add/remove + """ + if ":" not in param: + logger.warning( + "Param %s is missing a :, should be key:value pair. Skipping." % param + ) + return + + key, value = param.split(":", 1) + if command == "set": + self.set(key, value) + logger.info("Updated %s to be %s" % (key, value)) + elif command == "add": + self.add(key, value) + logger.info("Added %s to %s" % (key, value)) + elif command == "remove": + self.remove(key, value) + logger.info("Removed %s from %s" % (key, value)) + class Settings(SettingsBase): """ diff --git a/shpc/main/wrappers/base.py b/shpc/main/wrappers/base.py index d378b2e8e..c29e45377 100644 --- a/shpc/main/wrappers/base.py +++ b/shpc/main/wrappers/base.py @@ -3,7 +3,7 @@ __license__ = "MPL 2.0" from shpc.logger import logger -from jinja2 import Template, Environment, FileSystemLoader +from jinja2 import Environment, FileSystemLoader import shpc.utils import os diff --git a/shpc/settings.yml b/shpc/settings.yml index 1681ec771..2d38d622c 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -27,13 +27,20 @@ module_base: $root_dir/modules # This is where you might add a prefix to your module names, if desired. module_name: '{{ parsed_name.tool }}' -# Create a .version file for LMOD in the module folder +# When multiple versions are available and none requested, allow module picking one iself default_version: true # store containers separately from module files # It's recommended to do this for faster loading container_base: $root_dir/containers +# If defined, create a simplified "symlink install" that shortens module names +# to be just the container, e.g., ghcr.io/autamus/samtools -> module load samtools +symlink_base: $root_dir/symlinks + +# Always generate a symlink, even without the command line argument +symlink_tree: false + # if defined, add to lmod script to load this Singularity module first singularity_module: podman_module: diff --git a/shpc/tests/test_container.py b/shpc/tests/test_container.py index d4cc00537..77e976807 100644 --- a/shpc/tests/test_container.py +++ b/shpc/tests/test_container.py @@ -6,8 +6,6 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -import pytest -import shutil import os import shpc.main.container as container diff --git a/shpc/tests/test_container_config.py b/shpc/tests/test_container_config.py index 0092b7187..d5ad36ae3 100644 --- a/shpc/tests/test_container_config.py +++ b/shpc/tests/test_container_config.py @@ -6,8 +6,6 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -import pytest -import shutil import os import shpc.main.container as container diff --git a/shpc/tests/test_settings.py b/shpc/tests/test_settings.py index dfd632876..fd7d03bb6 100644 --- a/shpc/tests/test_settings.py +++ b/shpc/tests/test_settings.py @@ -7,7 +7,6 @@ # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. import pytest -import shutil import os from shpc.main.settings import Settings diff --git a/shpc/version.py b/shpc/version.py index eaa88df08..4f1f4e642 100644 --- a/shpc/version.py +++ b/shpc/version.py @@ -2,7 +2,7 @@ __copyright__ = "Copyright 2021-2022, Vanessa Sochat" __license__ = "MPL 2.0" -__version__ = "0.0.48" +__version__ = "0.0.49" AUTHOR = "Vanessa Sochat" NAME = "singularity-hpc" PACKAGE_URL = "https://github.com/singularityhub/singularity-hpc"