diff --git a/CHANGES.txt b/CHANGES.txt index fbe9bbde..429cddcc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,15 @@ + +v3.1.0 +users: + - Fixing an error related with the Plugin Manager when clicking the plugin treeview + - Fix workflow tutorial intro. + - New variable (flag) to define if binaries are installed by default or not --> SCIPION_DONT_INSTALL_BINARIES (any value will deactivate installation of binaries + - scipion3 config -p (or any other): shows plugin url too + - SetOfVolumes and SetOfParticles have DataViewer as the default one. + +developers: + Update the methods checkLib and addPackage to improve the installation of plugins with libraries dependencies. + v3.0.12 users: old config file is now backed up (at backup/scipion.conf.YYmmDDHHMMSS) always after any scipion config command that updates the config files. diff --git a/requirements.txt b/requirements.txt index 41136a58..a439c80a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ scipion-em -outdated==0.2.1 +outdated==0.2.2 diff --git a/scipion/__init__.py b/scipion/__init__.py index f61a7a16..4e2487c2 100644 --- a/scipion/__init__.py +++ b/scipion/__init__.py @@ -1,2 +1,2 @@ -__version__ = '3.0.12' +__version__ = '3.1.0' diff --git a/scipion/__main__.py b/scipion/__main__.py index af1d55fc..d76a5f2c 100755 --- a/scipion/__main__.py +++ b/scipion/__main__.py @@ -249,6 +249,8 @@ def main(): defaultViewers.append('"SetOfLandmarkModels":["imod.viewers.ImodViewer"]') defaultViewers.append('"SetOfTomograms":["imod.viewers.ImodViewer"]') defaultViewers.append('"SetOfSubTomograms":["pwem.viewers.DataViewer"]') + defaultViewers.append('"SetOfVolumes":["pwem.viewers.DataViewer"]') + defaultViewers.append('"SetOfParticles":["pwem.viewers.DataViewer"]') os.environ["VIEWERS"] = '{%s}' % ','.join(defaultViewers) diff --git a/scipion/install/funcs.py b/scipion/install/funcs.py index 018f9aa6..784708bc 100644 --- a/scipion/install/funcs.py +++ b/scipion/install/funcs.py @@ -24,6 +24,9 @@ # * # ************************************************************************** import logging + +from pyworkflow.utils import redStr + logger = logging.getLogger(__name__) import os import platform @@ -35,7 +38,7 @@ from pyworkflow import Config import pwem - +from typing import List, Tuple, Dict # Then we get some OS vars MACOSX = (platform.system() == 'Darwin') @@ -71,22 +74,15 @@ def checkLib(lib, target=None): stdout=open(os.devnull, 'w'), stderr=STDOUT) if ret != 0: raise OSError + return True except OSError: try: ret = call(['%s-config' % lib, '--cflags']) if ret != 0: raise OSError + return True except OSError: - print(""" - ************************************************************************ - Warning: %s not found. Please consider installing it first. - ************************************************************************ - -Continue anyway? (y/n)""" % lib) - if input().upper() != 'Y': - sys.exit(2) - # TODO: maybe write the result of the check in - # software/log/lib_...log so we don't check again if we already said "no" + return False class Command: @@ -116,7 +112,7 @@ def _existsAll(self): def execute(self): if not self._always and self._targets and self._existsAll(): print(" Skipping command: %s" % cyan(self._cmd)) - print(" All targets exist.") + print(" All targets %s exist." % self._targets) else: cwd = os.getcwd() if self._cwd is not None: @@ -155,8 +151,8 @@ def execute(self): if not self._env.showOnly: for t in self._targets: if not glob(t): - msg = "target '%s' not built (after running '%s')" % (t, cmd) - sys.exit(msg) + print(red("ERROR: File or folder '%s' not found after running '%s'." % (t, cmd))) + sys.exit(1) def __repr__(self): return self.__str__() @@ -213,7 +209,7 @@ def getName(self): def execute(self): t1 = time.time() - print(green("Building %s ..." % self._name)) + print(green("Installing %s ..." % self._name)) if not self._always and self._existsAll(): print(" All targets exist, skipping.") else: @@ -295,8 +291,8 @@ def getPythonPackagesFolder(): # import site # return site.getsitepackages()[0] - from distutils.sysconfig import get_python_lib - return get_python_lib() + from sysconfig import get_paths + return get_paths()["purelib"] @staticmethod def getIncludeFolder(): @@ -576,6 +572,7 @@ def addPackage(self, name, **kwargs): :param buildDir: Optional folder where build/extraction happens. If not passed will be inferred from tgz's name :param neededProgs: Optional, list of programs needed. E.g: make, cmake,... :param version: Optional, version of the package. + :param libChecks: Optional, a list of the libraries needed. E.g: libjpeg62, gsl (GSL - GNU Scientific Library) """ # Add to the list of available packages, for reference (used in --help). @@ -598,6 +595,21 @@ def addPackage(self, name, **kwargs): version = '' extName = name + # Check the required libraries + commands = kwargs.get('commands', []) + if 'libChecks' in kwargs: + cmdLibChecks = [] + libChecks = kwargs['libChecks'] + libChecks = list(libChecks) if type(libChecks) == str else libChecks + for libName in libChecks: + if not checkLib(libName): + msg = 'ERROR! Required library %s was not found. Please consider to install it ' \ + '(sudo apt-get install in Ubuntu, sudo yum install in centOS, etc).' % libName + cmdLibChecks.append(('echo "%s" && exit 1' % redStr(msg), libName)) + + if cmdLibChecks: + commands = cmdLibChecks + self._packages[name].append((name, version)) environ = (self.updateCudaEnviron(name) @@ -629,7 +641,6 @@ def addPackage(self, name, **kwargs): libArgs.update(kwargs) target = self._addDownloadUntar(extName, **libArgs) - commands = kwargs.get('commands', []) for cmd, tgt in commands: if isinstance(tgt, str): tgt = [tgt] @@ -863,3 +874,429 @@ def mkdir(path): if not exists(path): os.makedirs(path) return path + +class InstallHelper(): + """ + ### This class is intended to be used to ease the plugin installation process. + + #### Usage: + InstallHelper class needs to be instanciated before it can be used. + After that, commands can be chained together to run them in the defined order. + The last command always needs to be addPackage(). + + #### Example: + installer = InstallHelper() # Instanciating class\n + installer.getCloneCommand('test-package', '/home/user/myCustomPath', 'github.com/myRepo') # Cloning GitHub repository\n + installer.getCondaenvCommand('test-package') # Creating conda enviroment\n + installer.addPackage(env, 'test-package') # Install package\n + + #### It can also be done in a single line: + installer.getCloneCommand('test-package', '/home/user/myCustomPath', 'github.com/myRepo').getCondaenvCommand('test-package').addPackage(env, 'test-package')\n + + #### If you want to check the command strings you are producing, use the function getCommandList() instead of addPackage() and assign it to a variable so you can print it. + """ + # Global variables + DEFAULT_VERSION = '1.0' + + def __init__(self, packageName: str, packageHome: str=None, packageVersion: str=DEFAULT_VERSION): + """ + ### Constructor for the InstallHelper class. + + #### Parameters: + packageName (str): Name of the package. + packageHome (str): Optional. Path to the package. It can be absolute or relative to current directory. + packageVersion (str): Optional. Package version. + """ + # Private list of tuples containing commands with targets + self.__commandList = [] + + # Package name, version, and home + self.__packageName = packageName + self.__packageVersion = packageVersion + self.__packageHome = packageHome if packageHome else os.path.join(pwem.Config.EM_ROOT, packageName + '-' + packageVersion) + + #--------------------------------------- PRIVATE FUNCTIONS ---------------------------------------# + def __getTargetCommand(self, targetName: str) -> str: + """ + ### This private function returns the neccessary command to create a target file given its name. + ### Targets are always in uppercase and underscore format. + + #### Parameters: + targetName (str): Name of the target file. + + #### Returns: + (str): The command needed to create the target file. + """ + return 'touch {}'.format(targetName) + + def __getBinaryEnvName(self, binaryName: str, binaryVersion: str=DEFAULT_VERSION) -> str: + """ + ### This function returns the env name for a given package and repo. + + #### Parameters: + binaryName (str): Name of the binary inside the package. + binaryVersion (str): Optional. Binary's version. + + #### Returns: + (str): The enviroment name for this binary. + """ + return binaryName + "-" + binaryVersion + + def __getEnvActivationCommand(self, binaryName: str, binaryVersion: str=DEFAULT_VERSION) -> str: + """ + ### Returns the conda activation command for the given enviroment. + + #### Parameters: + binaryName (str): Name of the binary inside the package. + binaryVersion (str): Optional. Version of the binary inside the package. + + #### Returns: + (str): The enviroment activation command. + """ + return "conda activate " + self.__getBinaryEnvName(binaryName, binaryVersion=binaryVersion) + + def __getBinaryNameAndVersion(self, binaryName: str=None, binaryVersion: str=None) -> Tuple[str, str]: + """ + ### Returns the binary name and version from an optionally introduced binary name and version. + + #### Parameters: + binaryName (str): Name of the binary inside the package. + binaryVersion (str): Optional. Version of the binary inside the package. + + #### Returns: + tuple(str, str): The binary name and binary version. + """ + binaryName = binaryName if binaryName else self.__packageName + binaryVersion = binaryVersion if binaryVersion else self.__packageVersion + return binaryName, binaryVersion + + #--------------------------------------- PUBLIC FUNCTIONS ---------------------------------------# + def getCommandList(self) -> List[Tuple[str, str]]: + """ + ### This function returns the list of commands with targets for debugging purposes or to export into another install helper. + + #### Returns: + (list[tuple[str, str]]): Command list with target files. + + #### Usage: + commandList = installer.getCommandList() + """ + return self.__commandList + + def importCommandList(self, commandList: List[Tuple[str, str]]): + """ + ### This function inserts the given formatted commands from another install helper into the current one. + + #### Parameters: + commandList (list[tuple[str, str]]): List of commands generated by an install helper. + + #### Usage: + installer1 = InstallHelper('package1', packageHome='/home/user/package2', packageVersion='1.0') + installer1.addCommand('cd /home', 'CHANGED_DIRECTORY') + installer2 = InstallHelper('package2', packageHome='/home/user/package2', packageVersion='1.0') + installer2.importCommandList(installer1.getCommandList()) + + #### Note: + Argument 'packageHome' of the first installer must be the same as second installer. + """ + # Adding given commands to current list + self.__commandList.extend(commandList) + return self + + def addCommand(self, command: str, targetName: str, workDir: str=''): + """ + ### This function adds the given command with target to the command list. + ### The target file needs to be located inside packageHome's directory so Scipion can detect it. + + #### Parameters: + command (str): Command to be added. + targetName (str): Name of the target file to be produced after commands are completed successfully. + workDir (str): Optional. Directory where the command will be executed from. + + #### Usage: + installer.addCommand('python3 myScript.py', 'MYSCRIPT_COMPLETED', workDir='/home/user/Documents/otherDirectory') + + #### This function call will generate the following commands: + cd /home/user/Documents/otherDirectory && python3 myScript.py && touch /home/user/scipion/software/em/test-package-1.0/MYSCRIPT_COMPLETED + """ + # Getting work directory + workDirCmd = 'cd {} && '.format(workDir) if workDir else '' + + # Getting target name + fullTargetName = os.path.join(self.__packageHome, targetName) + + command = (workDirCmd + command) if workDir else command + self.__commandList.append((command + " && {}".format(self.__getTargetCommand(fullTargetName)), targetName)) + return self + + def addCommands(self, commandList: List[str], binaryName: str=None, workDir:str='', targetNames: List[str]=[]): + """ + ### This function adds the given commands with targets to the command list. + + #### Parameters: + commandList (list[str]): List containing the commands to add. + binaryName (str): Optional. Name of the binary. Default is package name. + workDir (str): Optional. Directory where the commands will be executed from. + targetNames (list[str]): Optional. List containing the name of the target files for this commands. + + #### Usage: + installer.addCommands(['python3 myScript.py', 'ls'], binaryName='myBinary', workDir='/home/user/Documents/otherDirectory', + targetNames=['MYSCRIPT_COMPLETED', 'DIRECTORY_LISTED']) + + #### This function call will generate the following commands: + cd /home/user/Documents/otherDirectory && python3 myScript.py && touch /home/user/scipion/software/em/test-package-1.0/MYSCRIPT_COMPLETED\n + cd /home/user/Documents/otherDirectory && ls && touch /home/user/scipion/software/em/test-package-1.0/DIRECTORY_LISTED + """ + # Defining binary name + binaryName = self.__getBinaryNameAndVersion(binaryName=binaryName)[0] + + # Defining default target name preffix + defaultTargetPreffix = '{}_EXTRA_COMMAND_'.format(binaryName.upper()) + + # Executing commands + for idx in range(len(commandList)): + targetName = targetNames[idx] if targetNames else (defaultTargetPreffix + str(idx)) + self.addCommand(commandList[idx], targetName, workDir=workDir) + + return self + + def getCloneCommand(self, url: str, binaryFolderName: str='', targeName: str=None): + """ + ### This function creates the neccessary command to clone a repository from Github. + + #### Parameters: + url (str): URL to the git repository. + binaryFolderName (str): Optional. Name of the binary directory. + targetName (str): Optional. Name of the target file for this command. + + #### Usage: + installer.getCloneCommand('https://github.com/myRepo.git', binaryFolderName='myCustomBinary', targeName='BINARY_CLONED') + + #### This function call will generate the following command: + cd /home/user/scipion/software/em/test-package-1.0 && git clone https://github.com/myRepo.git myCustomBinary && touch BINARY_CLONED + """ + # Defining target name + targeName = targeName if targeName else '{}_CLONED'.format(binaryFolderName.upper()) + + # Modifying binary name with a space for the command + binaryFolderName = (' ' + binaryFolderName) if binaryFolderName else '' + + # Adding command + self.addCommand('git clone {}{}'.format(url, binaryFolderName), targeName, workDir=self.__packageHome) + + return self + + def getCondaEnvCommand(self, binaryName: str=None, binaryPath: str=None, binaryVersion: str=None, pythonVersion: str=None, requirementsFile: bool=False, + requirementFileName: str='requirements.txt', requirementList: List[str]=[], extraCommands: List[str]=[], targetName: str=None): + """ + ### This function creates the command string for creating a Conda enviroment and installing required dependencies for a given binary inside a package. + + #### Parameters: + binaryName (str): Optional. Name of the binary. Default is package name. + binaryPath (str): Optional. Path to the binary. It can be absolute or relative to current directory. + binaryVersion (str): Optional. Binary's version. Default is package version. + pythonVersion (str): Optional. Python version needed for the package. + requirementsFile (bool): Optional. Defines if a Python requirements file exists. + requirementFileName (bool): Optional. Name of the Python requirements file. + requirementList (list[str]): Optional. List of Python packages to be installed. Can be used together with requirements file, but packages cannot be repeated. + extraCommands (list[str]): Optional. List of extra conda-related commands to execute within the conda enviroment. + targetName (str): Optional. Name of the target file for this command. + + #### Usage: + installer.getCondaEnvCommand(binaryName='myBinary', binaryPath='/home/user/scipion/software/em/test-package-1.0/myBinary', binaryVersion='1.5', pythonVersion='3.11', + requirementsFile=True, requirementFileName='requirements.txt', requirementList=['torch==1.2.0', 'numpy'], + extraCommands=['conda info --envs'], targetName='CONDA_ENV_CREATED') + + #### This function call will generate the following command: + eval "$(/home/user/miniconda/bin/conda shell.bash hook)"&& conda create -y -n myBinary-1.5 python=3.11 && conda activate myBinary-1.5 && + cd /home/user/scipion/software/em/test-package-1.0/myBinary && conda install pip -y && $CONDA_PREFIX/bin/pip install -r requirements.txt && + $CONDA_PREFIX/bin/pip install torch==1.2.0 numpyconda info --envs && cd /home/user/scipion/software/em/test-package-1.0 && touch CONDA_ENV_CREATED + #### The path in the first command (eval ...) might vary, depending on the value of CONDA_ACTIVATION_CMD in your scipion.conf file. + """ + # Binary name and version definition + binaryName, binaryVersion = self.__getBinaryNameAndVersion(binaryName=binaryName, binaryVersion=binaryVersion) + + # Conda env creation + createEnvCmd = 'conda create -y -n {}{}'.format(self.__getBinaryEnvName(binaryName, binaryVersion=binaryVersion), (' python={}'.format(pythonVersion)) if pythonVersion else '') + + # Command to install pip + pipInstallCmd = 'conda install pip -y' + + # Command prefix for Python packages installation + requirementPrefixCmd = '$CONDA_PREFIX/bin/pip install' + + # Requirements file name + requirementFileName = os.path.join(binaryPath, requirementFileName) if requirementFileName and binaryPath else requirementFileName + + # Command for installing Python packages with requirements file + installWithFile = (requirementPrefixCmd + ' -r ' + requirementFileName) if requirementsFile else '' + + # Command for installing Python packages manually + installManual = ' '.join(requirementList) + installManual = (requirementPrefixCmd + " " + installManual) if installManual else '' + + # Only install pip and Python packages if requiremenst file or manual list has been provided + pythonCommands = '' + if installWithFile or installManual: + pythonCommands = ' && ' + pipInstallCmd + pythonCommands += ' && {}'.format(installWithFile) if installWithFile else '' + pythonCommands += ' && {}'.format(installManual) if installManual else '' + + # Defining target name + targetName = targetName if targetName else '{}_CONDA_ENV_CREATED'.format(binaryName.upper()) + + # Crafting final command string + command = pwem.Plugin.getCondaActivationCmd() + ' ' + createEnvCmd # Basic commands: hook and env creation + command += ' && ' + self.__getEnvActivationCommand(binaryName, binaryVersion=binaryVersion) # Env activation + if binaryPath: + command += ' && cd {}'.format(binaryPath) # cd to binary path if proceeds + command += pythonCommands # Python related commands + if extraCommands: + command += " && " + " && ".join(extraCommands) # Extra conda commands + if binaryPath: + command += ' && cd {}'.format(self.__packageHome) # Return to package's root directory + + # Adding command + self.addCommand(command, targetName) + return self + + def addCondaPackages(self, packages: List[str], binaryName: str=None, binaryVersion: str=None, channel: str=None, targetName: str=None): + """ + ### This function returns the command used for installing extra packages in a conda enviroment. + + #### Parameters: + binaryName (str): Name of the binary. Default is package name. + packages (list[str]): List of conda packages to install. + binaryVersion (str): Optional. Binary's version. Default is package version. + channel (str): Optional. Channel to download the package from. + targetName (str): Optional. Name of the target file for this command. + + #### Usage: + installer.addCondaPackages(packages=['pytorch==1.1.0', 'cudatoolkit=10.0'], binaryName='myBinary', + binaryVersion='1.5', channel='conda-forge', targetName='CONDA_PACKAGES_INSTALLED') + + #### This function call will generate the following command: + eval "$(/home/user/miniconda/bin/conda shell.bash hook)"&& conda activate myBinary-1.5 && + conda install -y pytorch==1.1.0 cudatoolkit=10.0 -c conda-forge && touch CONDA_PACKAGES_INSTALLED + #### The path in the first command (eval ...) might vary, depending on the value of CONDA_ACTIVATION_CMD in your scipion.conf file. + """ + # Binary name and version definition + binaryName, binaryVersion = self.__getBinaryNameAndVersion(binaryName=binaryName, binaryVersion=binaryVersion) + + # Defininig target name + targetName = targetName if targetName else '{}_CONDA_PACKAGES_INSTALLED'.format(binaryName.upper()) + + # Adding installation command + command = "{} {} && conda install -y {}".format(pwem.Plugin.getCondaActivationCmd(), self.__getEnvActivationCommand(binaryName, binaryVersion=binaryVersion), ' '.join(packages)) + if channel: + command += " -c {}".format(channel) + self.addCommand(command, targetName) + + return self + + def getExtraFile(self, url: str, targetName: str, location: str=".", workDir: str='', fileName: str=None): + """ + ### This function creates the command to download with wget the file in the given link into the given path. + ### The downloaded file will overwrite a local one if they have the same name. + ### This is done to overwrite potential corrupt files whose download was not fully completed. + + #### Parameters: + url (str): URL of the resource to download. + targetName (str): Name of the target file for this command. + location (str): Optional. Location where the file will be downloaded. It can be absolute or relative to current directory. + workDir (str): Optional. Directory where the file will be downloaded from. + fileName (str): Optional. Name of the file after the download. Use intended for cases when expected name differs from url name. + + #### Usage: + installer.getExtraFile('https://site.com/myfile.tar', 'FILE_DOWNLOADED', location='/home/user/scipion/software/em/test-package-1.0/subdirectory', workDir='/home/user', fileName='test.tar') + + #### This function call will generate the following command: + cd /home/user && mkdir -p /home/user/scipion/software/em/test-package-1.0/subdirectory && + wget -O /home/user/scipion/software/em/test-package-1.0/subdirectory/test.tar https://site.com/myfile.tar && touch /home/user/scipion/software/em/test-package-1.0/FILE_DOWNLOADED + """ + # Getting filename for wget + fileName = fileName if fileName else os.path.basename(url) + mkdirCmd = "mkdir -p {} && ".format(location) if location else '' + + downloadCmd = "{}wget -O {} {}".format(mkdirCmd, os.path.join(location, fileName), url) + self.addCommand(downloadCmd, targetName, workDir=workDir) + + return self + + def getExtraFiles(self, fileList: List[Dict[str, str]], binaryName: str=None, workDir: str='', targetNames: List[str]=None): + """ + ### This function creates the command to download with wget the file in the given link into the given path. + ### The downloaded file will overwrite a local one if they have the same name. + ### This is done to overwrite potential corrupt files whose download was not fully completed. + + #### Parameters: + fileList (list[dict[str, str, str]]): List containing files to be downloaded. Example: [{'url': url1, 'path': path1, 'name': 'test.tar'}, {'url': url2, 'path': path2, 'name': 'test2.tar'}] + binaryName (str): Optional. Name of the binary. + Each file is a list contaning url and location to download it. Paths can be an empty string for default location. + workDir (str): Optional. Directory where the files will be downloaded from. + targetNames (list[str]): Optional. List containing the name of the target files for this commands. + + #### Usage: + installer.getExtraFiles( + [ + {'url': 'https://site.com/myfile.tar', 'path': '/home/user/scipion/software/em/test-package-1.0/subdirectory1', 'name': 'test.tar'}, + {'url': 'https://site.com/myfile.tar2', 'path': '/home/user/scipion/software/em/test-package-1.0/subdirectory2', 'name': 'test2.tar2'} + ], + binaryName='myBinary', workDir='/home/user', targetNames=['DOWNLOADED_FILE_1', 'DOWNLOADED_FILE_2']) + + #### This function call will generate the following commands: + cd /home/user && mkdir -p /home/user/scipion/software/em/test-package-1.0/subdirectory1 && + wget -O /home/user/scipion/software/em/test-package-1.0/subdirectory1/test.tar https://site.com/myfile.tar && touch /home/user/scipion/software/em/test-package-1.0/DOWNLOADED_FILE_1 + + cd /home/user && mkdir -p /home/user/scipion/software/em/test-package-1.0/subdirectory2 && + wget -O /home/user/scipion/software/em/test-package-1.0/subdirectory2/test2.tar2 https://site.com/myfile.tar2 && touch /home/user/scipion/software/em/test-package-1.0/DOWNLOADED_FILE_2 + """ + # Defining binary name + binaryName = self.__getBinaryNameAndVersion(binaryName=binaryName)[0] + + # Default preffix for target names + defaultTargetPreffix = "{}_FILE_".format(binaryName.upper()) + + # For each file in the list, download file + for idx in range(len(fileList)): + # Checking if file dictionary contains url + if 'url' not in fileList[idx]: + raise KeyError("ERROR: Download url has not been set for at least one file. You can create the appropiate dictionary calling function getFileDict.") + + # Getting proper file dictionary + kwargs = {} + if 'name' in fileList[idx]: + kwargs['name'] = fileList[idx]['name'] + if 'path' in fileList[idx]: + kwargs['path'] = fileList[idx]['path'] + downloadable = fileList[idx] if ('path' in fileList[idx] and 'name' in fileList[idx]) else self.getFileDict(fileList[idx]['url'], **kwargs) + + targetName = targetNames[idx] if targetNames else (defaultTargetPreffix + str(idx)) + self.getExtraFile(downloadable['url'], targetName, location=downloadable['path'], workDir=workDir, fileName=downloadable['name']) + + return self + + def addPackage(self, env, dependencies: List[str]=[], default: bool=True, **kwargs): + """ + ### This function adds the given package to scipion installation with some provided parameters. + + #### Parameters: + env: Scipion enviroment. + dependencies (list[str]): Optional. List of dependencies the package has. + default (bool): Optional. Defines if this package version is automatically installed with the plugin. + **kwargs: Optional. Other possible keyword parameters that will be directly passed to env.addPackage. + Intended for cases where multiple versions of the same package coexist in the same plugin. + + #### Usage: + installer.addPackage(env, dependencies=['wget', 'conda'], default=True) + """ + env.addPackage(self.__packageName, version=self.__packageVersion, tar='void.tgz', commands=self.__commandList, neededProgs=dependencies, default=default, **kwargs) + + #--------------------------------------- PUBLIC UTILS FUNCTIONS ---------------------------------------# + def getFileDict(self, url: str, path: str='.', fileName: str=None) -> Dict[str, str]: + """ This function generates the dictionary for a downloadable file. """ + # Getting file name + fileName = fileName if fileName else os.path.basename(url) + + # Returning dictionary + return {'url': url, 'path': path, 'name': fileName} diff --git a/scipion/install/install_plugin.py b/scipion/install/install_plugin.py index b9b1a767..e72c8b89 100644 --- a/scipion/install/install_plugin.py +++ b/scipion/install/install_plugin.py @@ -31,7 +31,7 @@ from scipion.constants import MODE_INSTALL_PLUGIN, MODE_UNINSTALL_PLUGIN from scipion.install import Environment -from scipion.install.plugin_funcs import PluginRepository, PluginInfo +from scipion.install.plugin_funcs import PluginRepository, PluginInfo, installBinsDefault from pyworkflow.utils import redStr # ************************************************************************ @@ -68,6 +68,7 @@ def installPluginMethods(): # Install parser # ############################################################################ + installParser = subparsers.add_parser(MODE_INSTALL_PLUGIN[1], aliases=[MODE_INSTALL_PLUGIN[0]], formatter_class=argparse.RawTextHelpFormatter, usage="%s [-h] [--noBin] [-p pluginName [pipVersion ...]]" % invokeCmd, @@ -171,6 +172,7 @@ def installPluginMethods(): parserUsed = modeToParser[mode] exitWithErrors = False + if parsedArgs.help or (mode in [MODE_INSTALL_BINS, MODE_UNINSTALL_BINS] and len(parsedArgs.binName) == 0): @@ -212,7 +214,7 @@ def installPluginMethods(): plugin = PluginInfo(pipName=pluginName, pluginSourceUrl=pluginSrc, remote=False) numberProcessor = parsedArgs.j installed = plugin.installPipModule() - if installed and not parsedArgs.noBin: + if installed and installBinsDefault() and not parsedArgs.noBin: plugin.getPluginClass()._defineVariables() plugin.installBin({'args': ['-j', numberProcessor]}) else: @@ -229,7 +231,8 @@ def installPluginMethods(): plugin = pluginDict.get(pluginName, None) if plugin: installed = plugin.installPipModule(version=pluginVersion) - if installed and not parsedArgs.noBin: + if installed and installBinsDefault() and not parsedArgs.noBin: + plugin.getPluginClass()._defineVariables() plugin.installBin({'args': ['-j', numberProcessor]}) else: print("WARNING: Plugin %s does not exist." % pluginName) @@ -237,14 +240,18 @@ def installPluginMethods(): elif parsedArgs.mode in MODE_UNINSTALL_PLUGIN: - for pluginName in parsedArgs.plugin: - plugin = PluginInfo(pluginName, pluginName, remote=False) - if plugin.isInstalled(): - if not parsedArgs.noBin: - plugin.uninstallBins() - plugin.uninstallPip() - else: - print("WARNING: Plugin %s is not installed." % pluginName) + if parsedArgs.plugin: + for pluginName in parsedArgs.plugin: + plugin = PluginInfo(pluginName, pluginName, remote=False) + if plugin.isInstalled(): + if installBinsDefault() and not parsedArgs.noBin: + plugin.uninstallBins() + plugin.uninstallPip() + else: + print("WARNING: Plugin %s is not installed." % pluginName) + else: + print("Incorrect usage of command 'uninstallp'. Execute 'scipion3 uninstallp --help' or " + "'scipion3 help' for more details.") elif parsedArgs.mode == MODE_INSTALL_BINS: binToInstallList = parsedArgs.binName diff --git a/scipion/install/plugin_funcs.py b/scipion/install/plugin_funcs.py index ffc529aa..7bf7b238 100644 --- a/scipion/install/plugin_funcs.py +++ b/scipion/install/plugin_funcs.py @@ -334,6 +334,8 @@ def getInstallenv(self, envArgs=None): plugin.defineBinaries(env) except Exception as e: print("Couldn't get binaries definition of %s plugin: %s" % (self.name, e)) + import traceback + traceback.print_exc() return env else: return None @@ -523,3 +525,10 @@ def ansi(n): else: printStr = "List of available plugins in plugin repository inaccessible at this time." return printStr + + +def installBinsDefault(): + """ Returns the default behaviour for installing binaries + By default it is TRUE, define "SCIPION_DONT_INSTALL_BINARIES" to anything to deactivate binaries installation""" + + return os.environ.get("SCIPION_DONT_INSTALL_BINARIES", True) == True \ No newline at end of file diff --git a/scipion/install/plugin_manager.py b/scipion/install/plugin_manager.py index 86495804..1d67aa09 100644 --- a/scipion/install/plugin_manager.py +++ b/scipion/install/plugin_manager.py @@ -673,22 +673,23 @@ def _onPluginTreeClick(self, event): self.popup_menu.unpost() x, y, widget = event.x, event.y, event.widget elem = widget.identify("element", x, y) - self.tree.selectedItem = self.tree.identify_row(y) - if "image" in elem: - # a box was clicked - self._treeOperation() - else: - if self.tree.selectedItem is not None: - if self.isPlugin(self.tree.item(self.tree.selectedItem, - "values")[0]): - self.showPluginInformation(self.tree.selectedItem) - else: - parent = self.tree.parent(self.tree.selectedItem) - self.showPluginInformation(parent) - if len(self.operationList.getOperations(None)): - self.executeOpsBtn.config(state='normal') - else: - self.executeOpsBtn.config(state='disable') + if elem: + self.tree.selectedItem = self.tree.identify_row(y) + if "image" in elem: + # a box was clicked + self._treeOperation() + else: + if self.tree.selectedItem is not None: + item = self.tree.selectedItem + if not self.isPlugin(self.tree.item(item, "values")[0]): + item = self.tree.parent(item) + + self.showPluginInformation(item) + + if len(self.operationList.getOperations(None)): + self.executeOpsBtn.config(state='normal') + else: + self.executeOpsBtn.config(state='disable') def _deleteSelectedOperation(self, e=None): """ @@ -887,7 +888,7 @@ def showPluginInformation(self, pluginName): plugin.latestRelease + ' available. Right-click on the ' 'plugin to update it)') self.topPanelTree.tag_configure('pluginVersion', - foreground=Color.RED_COLOR) + foreground=Config.SCIPION_MAIN_COLOR) else: self.topPanelTree.tag_configure('pluginVersion', foreground='black') self.topPanelTree.insert('', 'end', pluginVersion, diff --git a/scipion/scripts/config.py b/scipion/scripts/config.py index 7eecc1ec..25950551 100644 --- a/scipion/scripts/config.py +++ b/scipion/scripts/config.py @@ -103,6 +103,12 @@ def main(args=None): print("%s = %s" % (k, v)) print("\nThese variables can be added/edited in '%s'" % os.environ[SCIPION_CONFIG]) + url = plugin.getUrl() + + if url != "": + print("\nMore information these variables might be found at '%s'" + % url) + else: print("No plugin found with name '%s'. Module name is expected,\n" "i.e. 'scipion3 config -p xmipp3' shows the config variables " @@ -466,92 +472,6 @@ def compareConfigVariable(section, variableName, valueInConfig, valueInTemplate) yellow(valueInTemplate))) -# def guessJava(): -# """Guess the system's Java installation, return a dict with the Java keys""" -# -# options = {} -# candidates = [] -# -# # First check if the system has a favorite one. -# if 'JAVA_HOME' in os.environ: -# candidates.append(os.environ['JAVA_HOME']) -# -# # Add also all the ones related to a "javac" program. -# for d in os.environ.get('PATH', '').split(':'): -# if not os.path.isdir(d) or 'javac' not in os.listdir(d): -# continue -# javaBin = os.path.realpath(join(d, 'javac')) -# if javaBin.endswith('/bin/javac'): -# javaHome = javaBin[:-len('/bin/javac')] -# candidates.append(javaHome) -# if javaHome.endswith('/jre'): -# candidates.append(javaHome[:-len('/jre')]) -# -# # Check in order if for any of our candidates, all related -# # directories and files exist. If they do, that'd be our best guess. -# for javaHome in candidates: -# allExist = True -# for path in ['include', join('bin', 'javac'), join('bin', 'jar')]: -# if not exists(join(javaHome, path)): -# allExist = False -# if allExist: -# options['JAVA_HOME'] = javaHome -# break -# # We could instead check individually for JAVA_BINDIR, JAVAC -# # and so on, as we do with MPI options, but we go for an -# # easier and consistent case instead: everything must be under -# # JAVA_HOME, which is the most common case for Java. -# -# if not options: -# print(red("Warning: could not detect a suitable JAVA_HOME.")) -# if candidates: -# print(red("Our candidates were:\n %s" % '\n '.join(candidates))) -# -# return options -# -# -# def guessMPI(): -# """Guess the system's MPI installation, return a dict with MPI keys""" -# # Returns MPI_LIBDIR, MPI_INCLUDE and MPI_BINDIR as a dictionary. -# -# options = {} -# candidates = [] -# -# # First check if the system has a favorite one. -# for prefix in ['MPI_', 'MPI', 'OPENMPI_', 'OPENMPI']: -# if '%sHOME' % prefix in os.environ: -# candidates.append(os.environ['%sHOME' % prefix]) -# -# # Add also all the ones related to a "mpicc" program. -# for d in os.environ.get('PATH', '').split(':'): -# if not os.path.isdir(d) or 'mpicc' not in os.listdir(d): -# continue -# mpiBin = os.path.realpath(join(d, 'mpicc')) -# if 'MPI_BINDIR' not in options: -# options['MPI_BINDIR'] = os.path.dirname(mpiBin) -# if mpiBin.endswith('/bin/mpicc'): -# mpiHome = mpiBin[:-len('/bin/mpicc')] -# candidates.append(mpiHome) -# -# # Add some extra directories that are commonly around. -# candidates += ['/usr/lib/openmpi', '/usr/lib64/mpi/gcc/openmpi'] -# -# # Check in order if for any of our candidates, all related -# # directories and files exist. If they do, that'd be our best guess. -# for mpiHome in candidates: -# if (exists(join(mpiHome, 'include', 'mpi.h')) and -# 'MPI_INCLUDE' not in options): -# options['MPI_INCLUDE'] = join(mpiHome, 'include') -# if (exists(join(mpiHome, 'lib', 'libmpi.so')) and -# 'MPI_LIBDIR' not in options): -# options['MPI_LIBDIR'] = join(mpiHome, 'lib') -# if (exists(join(mpiHome, 'bin', 'mpicc')) and -# 'MPI_BINDIR' not in options): -# options['MPI_BINDIR'] = join(mpiHome, 'bin') -# -# return options -# - def getConfigPathFromConfigFile(scipionConfigFile, configFile): """ :param scipionConfigFile path to the config file to derive the folder name from diff --git a/scipion/scripts/kickoff.py b/scipion/scripts/kickoff.py index 66ca1ac7..739e2608 100644 --- a/scipion/scripts/kickoff.py +++ b/scipion/scripts/kickoff.py @@ -252,7 +252,7 @@ def _addPair(self, text, title, r, lf, widget=ENTRY, traceCallback=None, elif widget == CHECKBUTTON: var.set(YES if value else NO) widget = tk.Checkbutton(lf, text="", font=self.bigFont, variable=var, - onvalue=YES, offvalue=NO, bg="white") + onvalue=YES, offvalue=NO, bg=pw.Config.SCIPION_BG_COLOR) widget.grid(row=r, column=1, sticky='news', padx=(5, 10), pady=pady) @@ -438,9 +438,9 @@ def importTemplate(template, window): workflow = createTemplateFile(template) if workflow is not None: try: - window.getViewWidget().info('Importing the workflow...') + window.getViewWidget().info('Importing workflow %s' % workflow) window.project.loadProtocols(workflow) - window.getViewWidget().updateRunsGraph(True, reorganize=False) + window.getViewWidget().updateRunsGraph(True) window.getViewWidget().cleanInfo() except Exception as ex: window.showError(str(ex), exception=ex) diff --git a/scipion/templates/workflow_tutorial_intro.json b/scipion/templates/workflow_tutorial_intro.json index 5d94d8c6..0aa23132 100644 --- a/scipion/templates/workflow_tutorial_intro.json +++ b/scipion/templates/workflow_tutorial_intro.json @@ -4,6 +4,9 @@ "object.id": "2", "object.label": "1. import mics", "object.comment": "", + "_useQueue": false, + "_prerequisites": "", + "_queueParams": null, "runName": null, "runMode": 0, "importFrom": 0, @@ -21,13 +24,23 @@ "magnification": 56588, "samplingRateMode": 0, "samplingRate": 1.237, - "scannedPixelSize": 7.0 + "scannedPixelSize": 7.0, + "dataStreaming": false, + "timeout": 43200, + "fileTimeout": 30, + "blacklistDateFrom": null, + "blacklistDateTo": null, + "useRegexps": true, + "blacklistFile": null }, { "object.className": "XmippProtPreprocessMicrographs", - "object.id": "43", + "object.id": "57", "object.label": "2. downsample x5", "object.comment": "", + "_useQueue": false, + "_prerequisites": "", + "_queueParams": null, "runName": null, "runMode": 0, "orderComment": null, @@ -42,8 +55,16 @@ "doInvert": true, "doDownsample": true, "downFactor": 5.0, + "doDenoise": false, + "maxIteration": 50, "doSmooth": false, "sigmaConvolution": 2.0, + "doHighPass": false, + "highCutoff": 0.002, + "highRaised": 0.001, + "doLowPass": false, + "lowCutoff": 0.4, + "lowRaised": 0.001, "doNormalize": false, "hostName": "localhost", "numberOfThreads": 1, @@ -51,51 +72,69 @@ "inputMicrographs": "2.outputMicrographs" }, { - "object.className": "ProtCTFFind", - "object.id": "85", - "object.label": "3. ctffind3", + "object.className": "CistemProtCTFFind", + "object.id": "112", + "object.label": "3. ctffind4", "object.comment": "", + "_useQueue": false, + "_prerequisites": "", + "_queueParams": null, "runName": null, "runMode": 0, "recalculate": false, "sqliteFile": null, - "ctfDownFactor": 1.0, - "useCftfind4": true, + "inputType": 1, + "avgFrames": 3, + "usePowerSpectra": false, + "windowSize": 512, + "lowRes": 50.0, + "highRes": 10.0, + "minDefocus": 500.0, + "maxDefocus": 40000.0, + "stepDefocus": 500.0, + "slowSearch": false, + "fixAstig": true, "astigmatism": 100.0, "findPhaseShift": false, - "lowRes": 0.14, - "highRes": 0.46, - "minDefocus": 0.5, - "maxDefocus": 4.0, - "windowSize": 256, + "minPhaseShift": 0.0, + "maxPhaseShift": 180.0, + "stepPhaseShift": 10.0, "hostName": "localhost", "numberOfThreads": 4, "numberOfMpi": 1, - "inputMicrographs": "43.outputMicrographs" + "streamingWarning": null, + "streamingSleepOnWait": 0, + "streamingBatchSize": 1, + "inputMicrographs": "57.outputMicrographs" }, { "object.className": "XmippProtParticlePicking", - "object.id": "126", + "object.id": "170", "object.label": "4a. xmipp picking", "object.comment": "", + "_useQueue": false, + "_prerequisites": "", + "_queueParams": null, "runName": null, "runMode": 0, - "memory": 2.0, - "inputMicrographs": "43.outputMicrographs" + "saveDiscarded": false, + "doInteractive": true, + "inputMicrographs": "57.outputMicrographs" }, { "object.className": "XmippProtExtractParticles", - "object.id": "155", + "object.id": "205", "object.label": "5. extract particles", "object.comment": "", + "_useQueue": false, + "_prerequisites": "", + "_queueParams": null, "runName": null, "runMode": 0, - "micsSource": 0, + "downsampleType": 0, + "downFactor": 1.0, "boxSize": 110, - "doSort": true, - "rejectionMethod": 0, - "maxZscore": 3, - "percentage": 5, + "doBorders": false, "doRemoveDust": true, "thresholdDust": 3.5, "doInvert": false, @@ -103,31 +142,52 @@ "doNormalize": true, "normType": 2, "backRadius": -1, + "patchSize": -1, "hostName": "localhost", "numberOfThreads": 4, "numberOfMpi": 1, - "ctfRelations": "85.outputCTF" + "ctfRelations": "112.outputCTF", + "inputCoordinates": "170.outputCoordinates" }, { "object.className": "ProtImportVolumes", - "object.id": "200", + "object.id": "253", "object.label": "6. import vol", "object.comment": "", + "_useQueue": false, + "_prerequisites": "", + "_queueParams": null, "runName": null, "runMode": 0, "importFrom": 0, "filesPath": "", "filesPattern": "", "copyFiles": false, - "samplingRate": 6.185 + "emdbId": null, + "setHalfMaps": false, + "half1map": null, + "half2map": null, + "samplingRate": 6.185, + "setOrigCoord": false, + "x": null, + "y": null, + "z": null, + "dataStreaming": false, + "timeout": 43200, + "fileTimeout": 30 }, { "object.className": "XmippProtProjMatch", - "object.id": "230", + "object.id": "299", "object.label": "7. projection matching", "object.comment": "\n", + "_useQueue": false, + "_prerequisites": "", + "_queueParams": null, "runName": null, "runMode": 0, + "useGpu": true, + "gpuList": "0", "useInitialAngles": false, "cleanUpFiles": false, "doCTFCorrection": true, @@ -187,16 +247,25 @@ "hostName": "localhost", "numberOfThreads": 1, "numberOfMpi": 4, - "inputParticles": "155.outputParticles", - "input3DReferences": "200.outputVolume" + "inputParticles": "205.outputParticles", + "input3DReferences": "253.outputVolume" }, { "object.className": "EmanProtBoxing", - "object.id": "319", + "object.id": "395", "object.label": "4b Eman boxer", "object.comment": "\n", + "_useQueue": false, + "_prerequisites": "", + "_queueParams": null, "runName": null, "runMode": 0, - "inputMicrographs": "43.outputMicrographs" + "boxSize": -1, + "particleSize": -1, + "device": "cpu", + "invertY": false, + "hostName": "localhost", + "numberOfThreads": 1, + "inputMicrographs": "57.outputMicrographs" } -] +] \ No newline at end of file