Merge pull request #223 from scipion-em/devel

release 3.2.0

.github/workflows/python-package.yml

@@ -19,11 +19,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up MPI
-      uses: mpi4py/setup-mpi@v1
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: '3.8'
     - name: Install dependencies

CHANGES.txt

@@ -1,3 +1,12 @@
+3.2.0:
+  - add more possible outputs
+  - fixing ctf after aretomo
+  - make ctf auomatic estimation tolerant to errors
+  - odd/even processing implemented
+  - update FiducialModel
+  - changes in protocols to save if landmark model has residual information
+  - fix: interpolated flag is no set when correcting the CTFs
+  - user-controllable binning for display in 3dmod: plugin variable -> IMOD_VIEWER_BINNING = 1
 3.1.10:
   - fix DefocusTol param type for CTF correction protocol: must be integer
   - fix binning for other protocols

README.rst

@@ -91,10 +91,20 @@ Install the plugin in developer mode.
     scipion installp -p local/path/to/scipion-em-imod --devel
 
 
-IMOD binaries will be downloaded and installed automatically with the plugin, but you can also link an existing installation. Default installation path assumed is ``software/em/imod-4.11.24/IMOD``, if you want to change it, set *IMOD_HOME* in ``scipion.conf`` file to the folder where the IMOD is installed.
+IMOD binaries will be downloaded and installed automatically with the plugin, but you can also link an existing installation. Default installation path assumed is ``software/em/imod-4.11.24/IMOD``, if you want to change it, set ``IMOD_HOME`` in ``scipion.conf`` file to the folder where the IMOD is installed.
 
 To check the installation, simply run one of the tests. A complete list of tests can be displayed by executing ``scipion test --show --grep imod``
 
+**Changing the binning level for 3dmod**
+========================================
+For quick visualization purposes, it can be useful to bin images and tomograms on the fly using the ``ImodViewer`` provided by this plugin (i.e. ``3dmod``). For that, the user can define the ``IMOD_VIEWER_BINNING`` environment variable in ``scipion.conf`` as in the following example:
+
+.. code-block::
+        [PLUGINS]
+        
+        IMOD_HOME = %(EM_ROOT)s/imod-4.11.24/IMOD
+        IMOD_VIEWER_BINNING = 8
+
 Supported versions
 ------------------
 

imod/__init__.py

@@ -33,10 +33,11 @@
 from pyworkflow.gui.project.utils import OS
 import pwem
 
-from .constants import IMOD_HOME, ETOMO_CMD, DEFAULT_VERSION, VERSIONS
+from .constants import IMOD_HOME, ETOMO_CMD, DEFAULT_VERSION, VERSIONS, IMOD_VIEWER_BINNING
 
 
-__version__ = '3.1.10'
+
+__version__ = '3.2.0'
 _logo = "icon.png"
 _references = ['Kremer1996', 'Mastronarde2017']
 
@@ -55,6 +56,11 @@ class Plugin(pwem.Plugin):
     @classmethod
     def _defineVariables(cls):
         cls._defineEmVar(IMOD_HOME, cls._getIMODFolder(DEFAULT_VERSION))
+        cls._defineVar(IMOD_VIEWER_BINNING, 1)
+
+    @classmethod
+    def getViewerBinning(cls):
+        return cls.getVar(IMOD_VIEWER_BINNING)
 
     @classmethod
     def _getEMFolder(cls, version, *paths):

imod/constants.py

@@ -30,6 +30,7 @@
 # ----------------- Constants values ------------------------------------------
 
 IMOD_HOME = 'IMOD_HOME'
+IMOD_VIEWER_BINNING = 'IMOD_VIEWER_BINNING'
 ETOMO_CMD = 'etomo'
 
 VERSION_4_11_7 = '4.11.7'

imod/protocols/protocol_applyTransformationMatrix.py

@@ -34,7 +34,7 @@
 from tomo.objects import TiltSeries, TiltImage
 
 from .. import Plugin, utils
-from .protocol_base import ProtImodBase
+from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME
 
 
 class ProtImodApplyTransformationMatrix(ProtImodBase):
@@ -64,6 +64,14 @@ def _defineParams(self, form):
                            'binned by the given factor. Must be an integer '
                            'bigger than 1')
 
+        form.addParam('processOddEven',
+                      params.BooleanParam,
+                      expertLevel=params.LEVEL_ADVANCED,
+                      default=True,
+                      label='Apply to odd/even',
+                      help='If True, the full tilt series and the associated odd/even tilt series will be processed. '
+                           'The transformations applied to the odd/even tilt series will be exactly the same.')
+
     # -------------------------- INSERT steps functions -----------------------
     def _insertAllSteps(self):
         for ts in self.inputSetOfTiltSeries.get():
@@ -123,6 +131,16 @@ def computeAlignmentStep(self, tsObjId):
 
         Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment)
 
+        if self.applyToOddEven(ts):
+            oddFn = firstItem.getOdd().split('@')[1]
+            evenFn = firstItem.getEven().split('@')[1]
+            paramsAlignment['input'] = oddFn
+            paramsAlignment['output'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_ODD_NAME)
+            Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment)
+            paramsAlignment['input'] = evenFn
+            paramsAlignment['output'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_EVEN_NAME)
+            Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment)
+
     def generateOutputStackStep(self, tsObjId):
         output = self.getOutputInterpolatedSetOfTiltSeries(self.inputSetOfTiltSeries.get())
 
@@ -142,6 +160,8 @@ def generateOutputStackStep(self, tsObjId):
         if binning > 1:
             newTs.setSamplingRate(ts.getSamplingRate() * binning)
 
+        ih = ImageHandler()
+
         index = 1
         for tiltImage in ts:
             if tiltImage.isEnabled():
@@ -151,6 +171,13 @@ def generateOutputStackStep(self, tsObjId):
                 acq.setTiltAxisAngle(0.)
                 newTi.setAcquisition(acq)
                 newTi.setLocation(index, (os.path.join(extraPrefix, tiltImage.parseFileName())))
+                if self.applyToOddEven(ts):
+                    locationOdd = index, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME))
+                    locationEven = index, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME))
+                    newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)])
+                else:
+                    newTi.setOddEven([])
+
                 index += 1
                 if binning > 1:
                     newTi.setSamplingRate(tiltImage.getSamplingRate() * binning)

imod/protocols/protocol_auto3d.py

@@ -80,8 +80,6 @@ def _defineParams(self, form):
 
     # --------------------------- STEPS functions -----------------------------
     def processTiltSeriesStep(self, tsId):
-        ts = self._tsDict.getTs(tsId)
-
         workingFolder = self._getTmpPath(tsId)
         prefix = os.path.join(workingFolder, tsId)
 

imod/protocols/protocol_base.py

@@ -29,10 +29,12 @@
 from pyworkflow.object import Set, CsvList, Pointer
 from pyworkflow.protocol import STEPS_PARALLEL
 from pyworkflow.utils import path
+from pwem.emlib.image import ImageHandler
 from pwem.protocols import EMProtocol
 from tomo.protocols.protocol_base import ProtTomoBase, ProtTomoImportFiles
 from tomo.objects import (SetOfTiltSeries, SetOfTomograms, SetOfCTFTomoSeries,
-                          CTFTomoSeries, CTFTomo, SetOfTiltSeriesCoordinates)
+                          CTFTomoSeries, CTFTomo, SetOfTiltSeriesCoordinates,
+                          TiltSeries, TiltImage)
 
 from .. import Plugin, utils
 
@@ -45,6 +47,10 @@
 OUTPUT_CTF_SERIE = "CTFTomoSeries"
 OUTPUT_TOMOGRAMS_NAME = "Tomograms"
 OUTPUT_COORDINATES_3D_NAME = "Coordinates3D"
+EXT_MRCS_TS_EVEN_NAME = "_even.mrcs"
+EXT_MRCS_TS_ODD_NAME = "_odd.mrcs"
+EXT_MRC_EVEN_NAME = "_even.mrc"
+EXT_MRC_ODD_NAME = "_odd.mrc"
 
 
 class ProtImodBase(ProtTomoImportFiles, EMProtocol, ProtTomoBase):
@@ -80,7 +86,21 @@ def _defineImportParams(self, form):
         """ Method to define import params in protocol form """
         ProtTomoImportFiles._defineImportParams(self, form)
 
-    # --------------------------- CACULUS functions ---------------------------
+    # --------------------------- CALCULUS functions ---------------------------
+    def tryExceptDecorator(func):
+        """ This decorator wraps the step in a try/except module which adds
+        the tilt series ID to the failed TS array
+        in case the step fails"""
+
+        def wrapper(self, tsId, *args):
+            try:
+                func(self, tsId, *args)
+            except Exception as e:
+                self.error("Some error occurred calling %s with TS id %s: %s" % (func.__name__, tsId, e))
+                self._failedTs.append(tsId)
+
+        return wrapper
+
     def convertInputStep(self, tsObjId, generateAngleFile=True,
                          imodInterpolation=True, doSwap=False):
         """
@@ -199,7 +219,7 @@ def getBasicNewstackParams(self, ts, outputTsFileName, inputTsFileName=None,
         return argsAlignment, paramsAlignment
 
     # --------------------------- OUTPUT functions ----------------------------
-    def getOutputSetOfTiltSeries(self, inputSet, binning=1):
+    def getOutputSetOfTiltSeries(self, inputSet, binning=1) -> SetOfTiltSeries:
         """ Method to generate output classes of set of tilt-series"""
 
         if self.TiltSeries:
@@ -298,6 +318,7 @@ def getOutputFiducialModelNoGaps(self, tiltSeries=None):
 
             outputFiducialModelNoGaps.copyInfo(tiltSeries)
             outputFiducialModelNoGaps.setSetOfTiltSeries(tiltSeriesPointer)
+            outputFiducialModelNoGaps.setHasResidualInfo(True)
 
             outputFiducialModelNoGaps.setStreamState(Set.STREAM_OPEN)
 
@@ -315,6 +336,8 @@ def getOutputFiducialModelGaps(self):
 
             outputFiducialModelGaps.copyInfo(self.inputSetOfTiltSeries.get())
             outputFiducialModelGaps.setSetOfTiltSeries(self.inputSetOfTiltSeries)
+            outputFiducialModelGaps.setHasResidualInfo(False)
+
             outputFiducialModelGaps.setStreamState(Set.STREAM_OPEN)
 
 
@@ -365,6 +388,7 @@ def getOutputSetOfTomograms(self, inputSet, binning=1):
         if self.Tomograms:
             getattr(self, OUTPUT_TOMOGRAMS_NAME).enableAppend()
 
+
         else:
             outputSetOfTomograms = self._createSetOfTomograms()
 
@@ -463,7 +487,7 @@ def addCTFTomoSeriesToSetFromDefocusFile(self, inputTs, defocusFilePath, output)
                                                 flag=defocusFileFlag)
 
         else:
-            raise Exception(
+            raise ValueError(
                 f"Defocus file flag {defocusFileFlag} is not supported. Only supported formats "
                 "correspond to flags 0, 1, 4, 5, and 37.")
 
@@ -474,7 +498,7 @@ def addCTFTomoSeriesToSetFromDefocusFile(self, inputTs, defocusFilePath, output)
             newCTFTomo.setIndex(index)
 
             if index not in defocusUDict.keys() and index not in excludedViews:
-                raise Exception("ERROR IN TILT-SERIES %s: NO CTF ESTIMATED FOR VIEW %d, TILT ANGLE %f" % (
+                raise IndexError("ERROR IN TILT-SERIES %s: NO CTF ESTIMATED FOR VIEW %d, TILT ANGLE %f" % (
                     tsId, index, inputTs[index].getTiltAngle()))
 
             " Plain estimation (any defocus flag)"
@@ -539,7 +563,45 @@ def addCTFTomoSeriesToSetFromDefocusFile(self, inputTs, defocusFilePath, output)
 
         self._store()
 
+    def createOutputFailedSet(self, tsObjId):
+        # Check if the tilt-series ID is in the failed tilt-series
+        # list to add it to the set
+        if tsObjId in self._failedTs:
+            ts = self._getTiltSeries(tsObjId)
+            tsSet = self._getSetOfTiltSeries()
+            tsId = ts.getTsId()
+
+            output = self.getOutputFailedSetOfTiltSeries(tsSet)
+
+            newTs = TiltSeries(tsId=tsId)
+            newTs.copyInfo(ts)
+            output.append(newTs)
+
+            for index, tiltImage in enumerate(ts):
+                newTi = TiltImage()
+                newTi.copyInfo(tiltImage, copyId=True, copyTM=True)
+                newTi.setAcquisition(tiltImage.getAcquisition())
+                newTi.setLocation(tiltImage.getLocation())
+                if hasattr(self, "binning") and self.binning > 1:
+                    newTi.setSamplingRate(tiltImage.getSamplingRate() * self.binning.get())
+                newTs.append(newTi)
+
+            ih = ImageHandler()
+            x, y, z, _ = ih.getDimensions(newTs.getFirstItem().getFileName())
+            newTs.setDim((x, y, z))
+            newTs.write(properties=False)
+
+            output.update(newTs)
+            output.write()
+            self._store()
+
     # --------------------------- UTILS functions -----------------------------
+    def _getSetOfTiltSeries(self, pointer=False):
+        return self.inputSetOfTiltSeries.get() if not pointer else self.inputSetOfTiltSeries
+
+    def _getTiltSeries(self, itemId):
+        return self.inputSetOfTiltSeries.get()[itemId]
+
     def iterFiles(self):
         """ Iterate through the files matched with the pattern.
         Provide the fileName and fileId.
@@ -554,8 +616,8 @@ def iterFiles(self):
                 # this is set by the user by using #### format in the pattern
                 match = self._idRegex.match(fileName)
                 if match is None:
-                    raise Exception("File '%s' doesn't match the pattern '%s'"
-                                    % (fileName, self.getPattern()))
+                    raise ValueError("File '%s' doesn't match the pattern '%s'"
+                                     % (fileName, self.getPattern()))
 
                 fileId = int(match.group(1))
 
@@ -581,3 +643,6 @@ def _excludeByWords(self, files):
             allowedFiles.append(file)
 
         return allowedFiles
+
+    def applyToOddEven(self, ts):
+        return hasattr(self, "processOddEven") and self.processOddEven and ts.hasOddEven()