From 2593687d2411ed1c35717705f03ad8d49c058ac8 Mon Sep 17 00:00:00 2001 From: albertmena <12760268+albertmena@users.noreply.github.com> Date: Wed, 17 Jan 2024 08:22:21 +0100 Subject: [PATCH 01/12] deprecating deepdenoissing --- xmipp3/viewers/viewer.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/xmipp3/viewers/viewer.py b/xmipp3/viewers/viewer.py index f6ede2ae3..b0f4788bb 100644 --- a/xmipp3/viewers/viewer.py +++ b/xmipp3/viewers/viewer.py @@ -49,7 +49,6 @@ from xmipp3.protocols import XmippProtAngularGraphConsistency from xmipp3.protocols import XmippProtAssignmentTiltPair from xmipp3.protocols import XmippProtMovieGain -from xmipp3.protocols import XmippProtDeepDenoising from .plotter import XmippPlotter from xmipp3.protocols import XmippProtTiltAnalysis from pwem.viewers.viewers_data import MicrographsView @@ -76,7 +75,6 @@ class XmippViewer(DataViewer): XmippProtMultiRefAlignability, XmippProtAngularGraphConsistency, XmippProtMovieGain, - XmippProtDeepDenoising, XmippProtTiltAnalysis ] @@ -151,26 +149,6 @@ def _visualize(self, obj, **kwargs): self._views.append(xplotter) - elif issubclass(cls, XmippProtDeepDenoising): - fn = obj.outputParticles.getFileName() - self._views.append(ObjectView(self._project, obj.outputParticles.strId(), fn, - viewParams={VISIBLE: 'enabled id _filename ' - '_xmipp_corrDenoisedProjection ' - '_xmipp_corrDenoisedNoisy ' - '_xmipp_imageOriginal _xmipp_imageRef', - RENDER: '_filename _xmipp_imageOriginal ' - '_xmipp_imageRef', - SORT_BY: 'id', - MODE: MODE_MD})) - - md = emlib.MetaData(obj._getExtraPath('particlesDenoised.xmd')) - if (md.containsLabel(emlib.MDL_CORR_DENOISED_PROJECTION) and - md.containsLabel(emlib.MDL_CORR_DENOISED_NOISY)): - xplotter = XmippPlotter(windowTitle="denoised vs proj & denoised vs original") - xplotter.createSubPlot("Correlations", "corr_denoised_projection", "corr_denoised_original") - xplotter.plotScatterMd(md, mdLabelX=emlib.MDL_CORR_DENOISED_PROJECTION, - mdLabelY=emlib.MDL_CORR_DENOISED_NOISY ) - self._views.append(xplotter) elif issubclass(cls, XmippProtMovieGain): if obj.hasAttribute('outputGains'): From 22784233bb1f562f453825c6469c768740efe0e2 Mon Sep 17 00:00:00 2001 From: Carlos Oscar Sorzano Date: Fri, 19 Jan 2024 12:05:03 +0100 Subject: [PATCH 02/12] Removal of DeepGlobal --- xmipp3/protocols.conf | 1 - .../test_protocol_deep_center_assignment.py | 132 ------------------ 2 files changed, 133 deletions(-) delete mode 100644 xmipp3/tests/test_protocol_deep_center_assignment.py diff --git a/xmipp3/protocols.conf b/xmipp3/protocols.conf index 2cc56ed59..34014c747 100644 --- a/xmipp3/protocols.conf +++ b/xmipp3/protocols.conf @@ -99,7 +99,6 @@ Protocols SPA = [ {"tag": "protocol_group", "text": "Classify", "openItem": "False", "children": []}, {"tag": "protocol_group", "text": "Refine", "openItem": "False", "children": [ {"tag": "protocol", "value": "XmippProtReconstructHighRes", "text": "default"}, - {"tag": "protocol", "value": "XmippProtDeepGlobalAssignment", "text": "default"}, {"tag": "protocol", "value": "XmippProtProjMatch", "text": "default"}, {"tag": "protocol", "value": "XmippProtLocalCTF", "text": "default"}, {"tag": "section", "text": "more", "openItem": "False", "children": []} diff --git a/xmipp3/tests/test_protocol_deep_center_assignment.py b/xmipp3/tests/test_protocol_deep_center_assignment.py deleted file mode 100644 index 700b1d7b2..000000000 --- a/xmipp3/tests/test_protocol_deep_center_assignment.py +++ /dev/null @@ -1,132 +0,0 @@ -# ************************************************************************** -# * -# * Authors: Adrian Sansinena Rodriguez (adrian.sansinena@cnb.csic.es) -# * -# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC -# * -# * This program is free software; you can redistribute it and/or modify -# * it under the terms of the GNU General Public License as published by -# * the Free Software Foundation; either version 2 of the License, or -# * (at your option) any later version. -# * -# * This program is distributed in the hope that it will be useful, -# * but WITHOUT ANY WARRANTY; without even the implied warranty of -# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# * GNU General Public License for more details. -# * -# * You should have received a copy of the GNU General Public License -# * along with this program; if not, write to the Free Software -# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -# * 02111-1307 USA -# * -# * All comments concerning this program package may be sent to the -# * e-mail address 'scipion@cnb.csic.es' -# * -# ************************************************************************** - -################## QUALITY DISCLAIMER ################## -# This tests only proves that this protocol can successfully run without errors! -# It does not guarantee at all that the results will be any good. -# The input train set is the same as the set to predict, which causes artificially great results, -# that will get worse once real use cases are introduced. However, changing that makes no sense -# either, since the quality of the results cannot be checked, due to the non-deterministic nature -# of Neural Networks, specially regarding their training. - -# Scipion em imports -from pwem.protocols import ProtImportParticles, ProtSubSet, exists -from pyworkflow.tests import BaseTest, DataSet, setupTestProject - -# Plugin imports -from ..protocols import XmippProtDeepGlobalAssignment, XmippProtDeepCenter - -# ---------------------------- COMMON FUNCTIONS & VARIABLES ---------------------------- -genericError = "There was a problem while running this protocol." - -def runImportParticles(cls): - """ Import Particles. """ - args = {'importFrom': ProtImportParticles.IMPORT_FROM_SCIPION, - 'sqliteFile': cls.particles, - 'amplitudConstrast': 0.1, - 'sphericalAberration': 2.0, - 'voltage': 200, - 'samplingRate': 0.99, - 'haveDataBeenPhaseFlipped': True - } - - protImport = cls.newProtocol(ProtImportParticles, **args) - protImport.setObjLabel('import particles') - cls.launchProtocol(protImport) - return protImport - -def setUpClassBase(cls): - setupTestProject(cls) - - # Data - cls.dataset = DataSet.getDataSet('10010') - cls.particles = cls.dataset.getFile('particles') - cls.protImportParts = runImportParticles(cls) - -class TestDeepGlobalAssignment(BaseTest): - @classmethod - def setUpClass(cls): - setUpClassBase(cls) - - def test(self): - # Creating subset of particles (same for train and test) - subset = self.newProtocol(ProtSubSet, - inputFullSet=self.protImportParts.outputParticles, - chooseAtRandom=True, - nElements=400) - self.launchProtocol(subset) - - deepGA = self.newProtocol(XmippProtDeepGlobalAssignment, - inputImageSet=subset.outputParticles, - inputTrainSet=subset.outputParticles, - Xdim=128, - numModels=5, - numEpochs=1, - batchSize=32, - learningRate=0.001, - sigma=8, - patience=5) - self.launchProtocol(deepGA) - self.assertIsNotNone(deepGA.outputParticles, genericError) - -class TestDeepCenter(BaseTest): - @classmethod - def setUpClass(cls): - setUpClassBase(cls) - - def test(self): - subset = self.newProtocol(ProtSubSet, - inputFullSet=self.protImportParts.outputParticles, - chooseAtRandom=True, - nElements=400) - self.launchProtocol(subset) - - deepCenter = self.newProtocol(XmippProtDeepCenter, - inputImageSet=subset.outputParticles, - trainModels=True, - inputTrainSet=subset.outputParticles, - numModels=5, - numEpochs=1, - batchSize=32, - learningRate=0.001, - sigma=8, - patience=5) - self.launchProtocol(deepCenter) - - fnModel0 = deepCenter._getExtraPath("model0.h5") - fnModel2 = deepCenter._getExtraPath("model2.h5") - - self.assertTrue(exists(fnModel0), fnModel0 + " does not exist") - self.assertTrue(exists(fnModel2), fnModel2 + " does not exist") - - self.assertIsNotNone(deepCenter.outputParticles, genericError) - - deepCenter2 = self.newProtocol(XmippProtDeepCenter, - inputImageSet=subset.outputParticles, - trainModels=False) - self.launchProtocol(deepCenter2) - - self.assertIsNotNone(deepCenter2.outputParticles, genericError) From b20504dfc7b5fcc4e6f4cb2ae263023696758f32 Mon Sep 17 00:00:00 2001 From: Carlos Oscar Sorzano Date: Fri, 19 Jan 2024 12:05:31 +0100 Subject: [PATCH 03/12] Train and predict unified --- xmipp3/protocols/protocol_deep_center.py | 90 ++++++++++-------------- 1 file changed, 38 insertions(+), 52 deletions(-) diff --git a/xmipp3/protocols/protocol_deep_center.py b/xmipp3/protocols/protocol_deep_center.py index c22a96464..37661aaa9 100644 --- a/xmipp3/protocols/protocol_deep_center.py +++ b/xmipp3/protocols/protocol_deep_center.py @@ -29,6 +29,7 @@ IntParam, BooleanParam, GPU_LIST) from pyworkflow.protocol.constants import LEVEL_ADVANCED from pyworkflow.utils import Message +from pyworkflow.utils.path import createLink from pwem.protocols import ProtAlign2D from pwem.objects import String from xmipp3.convert import readSetOfParticles, writeSetOfParticles @@ -39,11 +40,11 @@ from pyworkflow import BETA, UPDATED, NEW, PROD class XmippProtDeepCenter(ProtAlign2D, xmipp3.XmippProtocol): - """Center a set of particles with a neural network.""" - _label = 'deep center' - _devStatus = BETA + """Center a set of particles in 2D using a neural network. The particles remain the same, but their alignment + includes an approximate shift to place them in the center.""" _lastUpdateVersion = VERSION_3_0 _conda_env = 'xmipp_DLTK_v1.0' + _label = 'deep center' def __init__(self, **args): ProtAlign2D.__init__(self, **args) @@ -61,32 +62,28 @@ def _defineParams(self, form): form.addSection(label=Message.LABEL_INPUT) - form.addParam('inputImageSet', PointerParam, label="Input Image set", + form.addParam('inputParticles', PointerParam, label="Input images", pointerClass='SetOfParticles', - help='The set of particles to center') + help='The set does not need to be centered or have alignment parameters') + form.addParam('sigma', FloatParam, label="Shift sigma", default=5, - expertLevel=LEVEL_ADVANCED, - help="Sigma for the training of the shift") - form.addParam('Xdim', IntParam, - label="Image size", - default=128, - expertLevel=LEVEL_ADVANCED, - help="Image size during the processing") + help="In pixels. This is used to generate artificially shifted particles.") - form.addSection('Training parameters') + form.addParam('precision', FloatParam, + label="Precision", + default=0.5, + help="In pixels.") - form.addParam('numModels', IntParam, - label="Number of models", default=5, - help="The maximum number of model available in xmipp is 5.") + form.addSection(label="Training") form.addParam('trainSetSize', IntParam, label="Train set size", default=5000, - help='How many particles from the training') + help='How many particles to use for training. Set to -1 for all of them') form.addParam('numEpochs', IntParam, label="Number of epochs", - default=10, + default=100, expertLevel=LEVEL_ADVANCED, help="Number of epochs for training.") @@ -104,58 +101,47 @@ def _defineParams(self, form): # --------------------------- INSERT steps functions -------------------------------------------- def _insertAllSteps(self): + self.fnImgs = self._getTmpPath('imgs.xmd') + self.fnImgsTrain = self._getTmpPath('imgsTrain.xmd') if self.useQueueForSteps() or self.useQueue(): myStr = os.environ["CUDA_VISIBLE_DEVICES"] else: myStr = self.gpuList.get() os.environ["CUDA_VISIBLE_DEVICES"] = self.gpuList.get() numGPU = myStr.split(',') - + self._insertFunctionStep("convertInputStep", self.inputParticles.get()) self._insertFunctionStep("train", numGPU[0]) self._insertFunctionStep("predict", numGPU[0]) - self._insertFunctionStep('createOutputStep') + self._insertFunctionStep("createOutputStep") # --------------------------- STEPS functions --------------------------------------------------- + def convertInputStep(self, inputSet): + writeSetOfParticles(inputSet, self.fnImgs) + if self.trainSetSize.get()>0: + self.runJob("xmipp_metadata_utilities","-i %s --operate random_subset %d -o %s"%\ + (self.fnImgs,self.trainSetSize, self.fnImgsTrain), numberOfMpi=1) + else: + createLink(self.fnImgs, self.fnImgsTrain) + def train(self, gpuId): - fnTrain = self._getTmpPath("trainingImages") - writeSetOfParticles(self.inputImageSet.get(), fnTrain+".xmd") - self.runJob("xmipp_metadata_utilities","-i %s.xmd --operate random_subset %d"%\ - (fnTrain,self.trainSetSize), numberOfMpi=1) - self.runJob("xmipp_image_resize", - "-i %s.xmd -o %s.stk --save_metadata_stack %s.xmd --fourier %d" % - (fnTrain, fnTrain, fnTrain, self.Xdim), - numberOfMpi=self.numberOfThreads.get() * self.numberOfMpi.get()) - args = "%s %s %f %d %d %s %d %f" %\ - (fnTrain+".xmd", self._getExtraPath("model"), self.sigma, - self.numEpochs, self.batchSize, gpuId, self.numModels, self.learningRate) + args = "-i %s --omodel %s --sigma %f --maxEpochs %d --batchSize %d --gpu %s --learningRate %f --precision %f"%\ + (self.fnImgsTrain, self._getExtraPath("model.h5"), self.sigma, self.numEpochs, self.batchSize, gpuId, + self.learningRate, self.precision) self.runJob(f"xmipp_deep_center", args, numberOfMpi=1, env=self.getCondaEnv()) def predict(self, gpuId): - fnPredict = self._getExtraPath("predictImages") - fnPredictResized = self._getTmpPath("predictImages") - writeSetOfParticles(self.inputImageSet.get(), fnPredict+".xmd") - self.runJob("xmipp_image_resize", - "-i %s.xmd -o %s.stk --save_metadata_stack %s.xmd --fourier %d" % - (fnPredict, fnPredictResized, fnPredictResized, self.Xdim), - numberOfMpi=self.numberOfThreads.get() * self.numberOfMpi.get()) - args = "%s %s %s %s" % ( - fnPredict+".xmd", gpuId, fnPredictResized+".xmd", self._getExtraPath("model")) + fnModel = self._getExtraPath("model.h5") + args = "-i %s --gpu %s --model %s -o %s" % (self.fnImgs, gpuId, fnModel, self._getExtraPath('particles.xmd')) self.runJob("xmipp_deep_center_predict", args, numberOfMpi=1, env=self.getCondaEnv()) def createOutputStep(self): - fnPredict = self._getExtraPath("predictImages.xmd") + fnPredict = self._getExtraPath("particles.xmd") outputSet = self._createSetOfParticles() readSetOfParticles(fnPredict, outputSet) - outputSet.copyInfo(self.inputImageSet.get()) - outputSet.setAlignmentProj() + outputSet.copyInfo(self.inputParticles.get()) + outputSet.setAlignment2D() self._defineOutputs(outputParticles=outputSet) self._store(outputSet) - self._defineSourceRelation(self.inputImageSet.get(), outputSet) - - # --------------------------- INFO functions -------------------------------- - def _methods(self): - methods = [] - if hasattr(self, 'outputParticles'): - methods.append("We learned a model to center particles from %i input images (%s)." \ - % (self.inputSet.get().getSize(), self.getObjectTag('inputSet'))) - return methods + self._defineSourceRelation(self.inputParticles.get(), outputSet) + + From 8f58f6a7b01b594fa4621a0fd03af8d4516a2b0d Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 26 Jan 2024 14:03:12 +0000 Subject: [PATCH 04/12] Initial commit --- .../protocols/protocol_reconstruct_fourier.py | 62 ++++++++++++++++--- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_fourier.py b/xmipp3/protocols/protocol_reconstruct_fourier.py index a9125b193..de4847367 100644 --- a/xmipp3/protocols/protocol_reconstruct_fourier.py +++ b/xmipp3/protocols/protocol_reconstruct_fourier.py @@ -26,13 +26,14 @@ from pwem.objects import Volume from pwem.protocols import ProtReconstruct3D +from pwem import emlib import pyworkflow.protocol.params as params import pyworkflow.protocol.constants as cons from xmipp3.convert import writeSetOfParticles from xmipp3.base import isXmippCudaPresent +from pyworkflow.utils import moveFile import os - class XmippProtReconstructFourier(ProtReconstruct3D): """ Reconstruct a volume using Xmipp_reconstruct_fourier from a given set of particles. @@ -68,6 +69,8 @@ def _defineParams(self, form): help='Maximum resolution (in Angstrom) to consider \n' 'in Fourier space (default Nyquist).\n' 'Param *--maxres* in Xmipp.') + form.addParam('useHalves', params.BooleanParam, label='Use halves', default=False, + help='Create separate reconstructions from two random subsets. Useful for resolution measurements') line = form.addLine('Padding factor', expertLevel=cons.LEVEL_ADVANCED, help='Padding of the input images. Higher number will result in more precise interpolation in Fourier ' @@ -102,21 +105,35 @@ def _createFilenameTemplates(self): """ Centralize how files are called for iterations and references. """ myDict = { 'input_xmd': self._getExtraPath('input_particles.xmd'), - 'output_volume': self._getPath('output_volume.mrc') + 'half1_xmd': self._getExtraPath('input_particles000001.xmd'), + 'half2_xmd': self._getExtraPath('input_particles000002.xmd'), + 'output_volume': self._getPath('output_volume.mrc'), + 'half1_volume': self._getPath('half1.mrc'), + 'half2_volume': self._getPath('half2.mrc') } self._updateFilenamesDict(myDict) def _insertAllSteps(self): self._createFilenameTemplates() self._insertFunctionStep('convertInputStep') - self._insertReconstructStep() + if self.useHalves.get(): + self._insertFunctionStep('splitInputStep') + self._insertReconstructStep('half1') + self._insertReconstructStep('half2') + else: + self._insertReconstructStep() self._insertFunctionStep('createOutputStep') - def _insertReconstructStep(self): + def _insertReconstructStep(self, half=None): #imgSet = self.inputParticles.get() - params = ' -i %s' % self._getFileName('input_xmd') - params += ' -o %s' % self._getFileName('output_volume') + if half is None: + params = ' -i %s' % self._getFileName('input_xmd') + params += ' -o %s' % self._getFileName('output_volume') + else: + params = ' -i %s' % self._getFileName(half + '_xmd') + params += ' -o %s' % self._getFileName(half + '_volume') + params += ' --sym %s' % self.symmetryGroup.get() maxRes = self.maxRes.get() if maxRes == -1: @@ -163,7 +180,14 @@ def convertInputStep(self): #TODO: This only writes metadata what about binary file #it should writeSetOfParticles(imgSet, particlesMd) + + def splitInputStep(self): + args = [] + args += ['-i', self._getFileName('input_xmd')] + args += ['-n', 2] + self.runJob('xmipp_metadata_split', args, numberOfMpi=1) + def reconstructStep(self, params): """ Create the input file in STAR format as expected by Xmipp. If the input particles comes from Xmipp, just link the file. @@ -178,16 +202,34 @@ def reconstructStep(self, params): self.runJob('xmipp_reconstruct_fourier', params) else: self.runJob('xmipp_reconstruct_fourier_accel', params) - - self.runJob("xmipp_image_header", "-i %s --sampling_rate %f"%\ - (self._getFileName('output_volume'), self.inputParticles.get().getSamplingRate()), - numberOfMpi=1) + def averageStep(self): + # Read + half1 = emlib.Image(self._getFileName('half1_volume')) + half2 = emlib.Image(self._getFileName('half2_volume')) + + # Average + half1.inplaceAdd(half2) + half1.inplaceMultiply(0.5) + + # Write + half1.write(self._getFileName('output_volume')) + def createOutputStep(self): imgSet = self.inputParticles.get() + + self.runJob("xmipp_image_header", "-i %s --sampling_rate %f"%\ + (self._getFileName('output_volume'), imgSet.getSamplingRate()), + numberOfMpi=1) + volume = Volume() volume.setFileName(self._getFileName('output_volume')) volume.setSamplingRate(imgSet.getSamplingRate()) + if self.useHalves.get(): + volume.setHalfMaps([ + self._getFileName('half1_volume'), + self._getFileName('half2_volume') + ]) self._defineOutputs(outputVolume=volume) self._defineSourceRelation(self.inputParticles, volume) From 24cd44fb85399ff0594f08bd47386c728e80f422 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 26 Jan 2024 15:10:39 +0000 Subject: [PATCH 05/12] Bugfix --- xmipp3/protocols/protocol_reconstruct_fourier.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xmipp3/protocols/protocol_reconstruct_fourier.py b/xmipp3/protocols/protocol_reconstruct_fourier.py index de4847367..056f66d73 100644 --- a/xmipp3/protocols/protocol_reconstruct_fourier.py +++ b/xmipp3/protocols/protocol_reconstruct_fourier.py @@ -120,6 +120,7 @@ def _insertAllSteps(self): self._insertFunctionStep('splitInputStep') self._insertReconstructStep('half1') self._insertReconstructStep('half2') + self._insertFunctionStep('averageStep') else: self._insertReconstructStep() self._insertFunctionStep('createOutputStep') From edc5facd49fc401dc24703f17ef93d7fa7851425 Mon Sep 17 00:00:00 2001 From: James Krieger Date: Thu, 1 Feb 2024 10:36:04 +0100 Subject: [PATCH 06/12] handle either input map with same attribute inputMap --- xmipp3/protocols/protocol_resolution_bfactor.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/xmipp3/protocols/protocol_resolution_bfactor.py b/xmipp3/protocols/protocol_resolution_bfactor.py index 786bea3eb..f3eb1ebf7 100644 --- a/xmipp3/protocols/protocol_resolution_bfactor.py +++ b/xmipp3/protocols/protocol_resolution_bfactor.py @@ -137,8 +137,13 @@ def mrc_convert(self, fileName, outputFileName): def convertInputStep(self): """ Read the input volume and check the file extension to convert to mrc is it is the case. """ - self.vol = self.mrc_convert(self.localResolutionMap.get().getFileName(), - self._getTmpPath('localResolutionMap.mrc')) + if self.normalizeResolution.get(): + self.inputMap = self.localResolutionMap + else: + self.inputMap = self.normalizedMap + + self.vol = self.mrc_convert(self.inputMap.get().getFileName(), + self._getTmpPath('localResolutionMap.mrc')) def matchingBfactorLocalResolution(self): """ The local resolution map and the pdb are taken and analyzed to match the @@ -151,7 +156,7 @@ def matchingBfactorLocalResolution(self): params += ' --vol %s' % self.vol if self.normalizeResolution.get(): params += ' --fscResolution %s' % self.fscResolution.get() - params += ' --sampling %f' % self.localResolutionMap.get().getSamplingRate() + params += ' --sampling %f' % self.inputMap.get().getSamplingRate() if self.medianEstimation.get(): params += ' --useMedian ' if self.centered.get(): From 97799cc14e030ce0bae1e465599823eaaa49c685 Mon Sep 17 00:00:00 2001 From: James Krieger Date: Thu, 1 Feb 2024 10:39:34 +0100 Subject: [PATCH 07/12] fix help spaces --- xmipp3/protocols/protocol_resolution_bfactor.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/xmipp3/protocols/protocol_resolution_bfactor.py b/xmipp3/protocols/protocol_resolution_bfactor.py index f3eb1ebf7..7b13673e7 100644 --- a/xmipp3/protocols/protocol_resolution_bfactor.py +++ b/xmipp3/protocols/protocol_resolution_bfactor.py @@ -70,13 +70,13 @@ def _defineParams(self, form): form.addParam('normalizeResolution', BooleanParam, default=True, label="Normalize Resolution", - help='The normalizedlocal resolution map is defined as' - '(LR - FSC)/FSC, where LR is the local resolution of a' + help='The normalizedlocal resolution map is defined as ' + '(LR - FSC)/FSC, where LR is the local resolution of a ' 'given voxel, and FSC is the FSC resolution in A. This ' - 'map provides information about whether the local resolution is' + 'map provides information about whether the local resolution is ' 'greater or lesser than the FSC. The local resolution ' - 'normalized map is used to carry out the matching with the local' - 'bfactor per residue. Yes means that the local resolution will be' + 'normalized map is used to carry out the matching with the local ' + 'bfactor per residue. Yes means that the local resolution will be ' 'normalized by the algorithm. No means that the input local ' 'resolution map is already a normalized local resolution map.') @@ -85,7 +85,7 @@ def _defineParams(self, form): condition='normalizeResolution', help='Select a local resolution map. Alternatively, the input.' ' can be a normalized local resolution map, in this case' - 'set the Normalize resolution to No') + ' set the Normalize resolution to No') form.addParam('normalizedMap', PointerParam, pointerClass='Volume', label="Normalized Local Resolution Map", important=True, @@ -93,7 +93,7 @@ def _defineParams(self, form): help='Select a normalized local resolution map. The local' ' resolution normalized map is defined as ' '(LR - FSC)/FSC, where LR is the local resolution of a' - 'given voxel, and FSC is the FSC resolution in A') + ' given voxel, and FSC is the FSC resolution in A') form.addParam('fscResolution', FloatParam, condition = 'normalizeResolution', @@ -102,7 +102,7 @@ def _defineParams(self, form): form.addParam('medianEstimation', BooleanParam, default=True, label="Use median", - help='The local resolution per residue can be estimated using' + help='The local resolution per residue can be estimated using ' 'the mean (by default - No) or the median (yes)') form.addParam('centered', BooleanParam, default=True, From c0ab1de747d6f7d83c35b21cd0cc9bba90733e7c Mon Sep 17 00:00:00 2001 From: James Krieger Date: Thu, 1 Feb 2024 10:48:09 +0100 Subject: [PATCH 08/12] update resolution bfac test --- xmipp3/tests/test_protocols_xmipp_3d.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/xmipp3/tests/test_protocols_xmipp_3d.py b/xmipp3/tests/test_protocols_xmipp_3d.py index 77c55edb8..99bf1a7ca 100644 --- a/xmipp3/tests/test_protocols_xmipp_3d.py +++ b/xmipp3/tests/test_protocols_xmipp_3d.py @@ -2754,6 +2754,14 @@ def testXmippResolutionBfactor(self): fscResolution=8.35) self.launchProtocol(protbfactorResolution) + # Protocol local resolution/local bfactor with assuming already normalized + print("Protocol local resolution/local bfactor") + protbfactorResolution = self.newProtocol(XmippProtbfactorResolution, + normalizeResolution=False, + pdbfile=protImportPdb.outputPdb, + normalizedMap=protCreateMap.resolution_Volume) + self.launchProtocol(protbfactorResolution) + class TestXmippLocalVolAdjust(TestXmippBase): @classmethod From b373f67533326119982a84af2ea2174004669e2e Mon Sep 17 00:00:00 2001 From: Daniel Marchan Date: Fri, 22 Mar 2024 09:57:27 +0000 Subject: [PATCH 09/12] fix update DoseAnalysis protocol --- .../protocols/protocol_movie_dose_analysis.py | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/xmipp3/protocols/protocol_movie_dose_analysis.py b/xmipp3/protocols/protocol_movie_dose_analysis.py index 200190c37..90c58fe14 100644 --- a/xmipp3/protocols/protocol_movie_dose_analysis.py +++ b/xmipp3/protocols/protocol_movie_dose_analysis.py @@ -27,9 +27,9 @@ import matplotlib.pyplot as plt import numpy as np import os -from pyworkflow import VERSION_3_0, NEW +from pyworkflow import VERSION_3_0 from pyworkflow.object import Set -from pyworkflow.protocol import STEPS_PARALLEL +from pyworkflow.protocol import Protocol from pyworkflow.protocol.params import (PointerParam, IntParam, FloatParam, LEVEL_ADVANCED) from pyworkflow.utils.properties import Message import pyworkflow.protocol.constants as cons @@ -37,14 +37,13 @@ from pwem.objects import SetOfMovies from pwem.protocols import ProtProcessMovies from xmipp3.convert import getScipionObj -import statistics as stat -from pyworkflow import BETA, UPDATED, NEW, PROD +from pyworkflow import NEW, PROD THRESHOLD = 2 OUTPUT_ACCEPTED = 'outputMovies' OUTPUT_DISCARDED = 'outputMoviesDiscarded' -class XmippProtMovieDoseAnalysis(ProtProcessMovies): +class XmippProtMovieDoseAnalysis(ProtProcessMovies, Protocol): """ Protocol for the dose analysis """ # FIXME: WITH .mrcs IT DOES NOT FILL THE LABELS _devStatus = PROD @@ -311,24 +310,6 @@ def _checkNewOutput(self): if outputStep and outputStep.isWaiting(): outputStep.setStatus(cons.STATUS_NEW) - def _updateOutputSet(self, outputName, outputSet, state=Set.STREAM_OPEN): - outputSet.setStreamState(state) - if self.hasAttribute(outputName): - outputSet.write() # Write to commit changes - outputAttr = getattr(self, outputName) - # Copy the properties to the object contained in the protocol - outputAttr.copy(outputSet, copyId=False) - # Persist changes - self._store(outputAttr) - else: - # Here the defineOutputs function will call the write() method - self._defineOutputs(**{outputName: outputSet}) - self._store(outputSet) - - # Close set databaset to avoid locking it - outputSet.close() - - # ------------------------- UTILS functions -------------------------------- def getLimitIntervals(self): """ Funtion to obtain the acceptance interval limits.""" From f89c9c2eccd68643c135b4ff60f746e70e52ffca Mon Sep 17 00:00:00 2001 From: Daniel Marchan Date: Fri, 22 Mar 2024 10:30:03 +0000 Subject: [PATCH 10/12] fix indirect pointer when creating the 2D Classes --- xmipp3/protocols/protocol_core_analysis.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/xmipp3/protocols/protocol_core_analysis.py b/xmipp3/protocols/protocol_core_analysis.py index caeed6ba6..cd1a7f30a 100644 --- a/xmipp3/protocols/protocol_core_analysis.py +++ b/xmipp3/protocols/protocol_core_analysis.py @@ -30,8 +30,7 @@ import pyworkflow.protocol.params as param -import pyworkflow.protocol.constants as const -from pyworkflow.utils.path import cleanPath, makePath +from pyworkflow.utils.path import makePath import pwem.emlib.metadata as md from pwem.protocols import ProtClassify2D @@ -91,11 +90,11 @@ def analyzeCore(self): inputMdName = join(fnLevel, 'level_classes.xmd') writeSetOfClasses2D(self.inputClasses.get(), inputMdName, writeParticles=True) - args = " --dir %s --root level --computeCore %f %f"%(self._getExtraPath(), - self.thZscore, self.thPCAZscore) + args = " --dir %s --root level --computeCore %f %f" % (self._getExtraPath(), + self.thZscore, self.thPCAZscore) self.runJob('xmipp_classify_CL2D_core_analysis', args) self.runJob("xmipp_classify_evaluate_classes", "-i %s"%\ - self._getExtraPath(join("level_00","level_classes_core.xmd")), numberOfMpi=1) + self._getExtraPath(join("level_00", "level_classes_core.xmd")), numberOfMpi=1) #--------------------------- STEPS functions ------------------------------- def _defineFileNames(self): @@ -104,8 +103,8 @@ def _defineFileNames(self): myDict = { 'final_classes': self._getPath('classes2D%(sub)s.sqlite'), 'output_particles': self._getExtraPath('images.xmd'), - 'level_classes' : self.levelPath + 'level_classes%(sub)s.xmd', - 'level_images' : self.levelPath + 'level_images%(sub)s.xmd', + 'level_classes': self.levelPath + 'level_classes%(sub)s.xmd', + 'level_images': self.levelPath + 'level_images%(sub)s.xmd', 'classes_scipion': (self.levelPath + 'classes_scipion_level_' '%(level)02d%(sub)s.sqlite'), } @@ -115,7 +114,7 @@ def createOutputStep(self): """ Store the SetOfClasses2D object resulting from the protocol execution. """ - inputParticles = self.inputClasses.get().getImages() + inputParticles = self.inputClasses.get().getImagesPointer() level = self._getLastLevel() subset = CLASSES_CORE @@ -143,7 +142,7 @@ def _methods(self): strline ='We calculated the class cores %s. [Sorzano2014]' % self.getObjectTag('outputClasses_core') return [strline] - #--------------------------- UTILS functions ------------------------------- + # --------------------------- UTILS functions ------------------------------- def _updateParticle(self, item, row): item.setClassId(row.getValue(md.MDL_REF)) item.setTransform(rowToAlignment(row, ALIGN_2D)) From 4853b12a7461b95984f5d32c580b7e0c40f868e5 Mon Sep 17 00:00:00 2001 From: Daniel Marchan Date: Fri, 22 Mar 2024 11:34:40 +0000 Subject: [PATCH 11/12] automatic selection of window size tilt analysis protocol --- xmipp3/protocols/protocol_tilt_analysis.py | 33 +++++++++++++++++++--- xmipp3/tests/test_protocols_xmipp_mics.py | 4 +-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/xmipp3/protocols/protocol_tilt_analysis.py b/xmipp3/protocols/protocol_tilt_analysis.py index debb95d27..3537f9d31 100644 --- a/xmipp3/protocols/protocol_tilt_analysis.py +++ b/xmipp3/protocols/protocol_tilt_analysis.py @@ -44,8 +44,8 @@ from pwem.protocols import ProtMicrographs from xmipp3 import emlib from xmipp3.convert import getScipionObj -from matplotlib import pyplot as plt +AUTOMATIC_WINDOW_SIZES = [4096, 2048, 1024, 512, 256] class XmippProtTiltAnalysis(ProtMicrographs, Protocol): """ Estimate the tilt of a micrograph, by analyzing the PSD correlations of different segments of the image. @@ -67,10 +67,17 @@ def _defineParams(self, form): label="Input micrographs", important=True, help='Select the SetOfMicrograph to be preprocessed.') + form.addParam('autoWindow', BooleanParam, default=True, expertLevel=LEVEL_ADVANCED, + label='Estimate automatically the window size?', + help='Use this button to decide if you want to estimate the window size' + 'based on the movie size. It will select between 512, 1024, 2048 or 4096. ' + 'Condition: window_size <= Size/2.5 ') + form.addParam('window_size', IntParam, label='Window size', + condition="not autoWindow", default=1024, expertLevel=LEVEL_ADVANCED, validators=[GE(256, 'Error must be greater than 256')], - help='''By default, the micrograph will be divided into windows of dimensions 512x512, the PSD ''' - '''its correlations will be computed in every segment.''') + help=''' By default, the micrograph will be divided into windows of dimensions 1024x1024, ''' + ''' the PSD its correlations will be computed in every segment.''') form.addParam('objective_resolution', FloatParam, label='Objective resolution', default=3, expertLevel=LEVEL_ADVANCED, validators=[GT(0, 'Error must be Positive')], @@ -118,6 +125,7 @@ def initializeStep(self): self.streamClosed = self.inputMicrographs.get().isStreamClosed() self.micsDict = {mic.getObjId(): mic.clone() for mic in self._loadInputList(self.micsFn).iterItems()} pwutils.makePath(self._getExtraPath('DONE')) + self.windowSize = self.getWindowSize() def createOutputStep(self): self._closeOutputSet() @@ -337,7 +345,7 @@ def calculateTiltCorrelationStep(self, mic): micFolder = self._getOutputMicFolder(mic) micImage = ImageHandler().read(mic.getLocation()) dimx, dimy, z, n = micImage.getDimensions() - wind_step = self.window_size.get() + wind_step = self.windowSize x_steps, y_steps = window_coordinates2D(dimx, dimy, wind_step) subWindStep = int(wind_step * (self.samplingRate / self.objective_resolution.get())) x_steps_psd, y_steps_psd = window_coordinates2D(subWindStep * 2, subWindStep * 2, subWindStep) @@ -472,6 +480,23 @@ def _correctFormat(self, micName, micFn, micFolderTmp): return newMicName + def getWindowSize(self): + """ Function to get the window size, automatically or the one set by the user. """ + if self.autoWindow: + dimX, dimY, _ = self.getInputMicrographs().getFirstItem().getDimensions() + halfMic = int(min(dimX, dimY)/2.5) + windowSize = halfMic # In case there is not a suitable option, very unlikely + for sizeAuto in AUTOMATIC_WINDOW_SIZES: + if sizeAuto < halfMic: # Exit in case you found a suitable option + windowSize = sizeAuto + break + else: + windowSize = self.window_size.get() + + self.info('The window size used is %d' % windowSize) + + return windowSize + def _insertFinalSteps(self, deps): """ This should be implemented in subclasses""" return deps diff --git a/xmipp3/tests/test_protocols_xmipp_mics.py b/xmipp3/tests/test_protocols_xmipp_mics.py index 098a65a0c..f7903238f 100644 --- a/xmipp3/tests/test_protocols_xmipp_mics.py +++ b/xmipp3/tests/test_protocols_xmipp_mics.py @@ -1185,7 +1185,7 @@ def setUpClass(cls): magnification=56000) def testTiltAnalysis(self): - protTilt = XmippProtTiltAnalysis(window_size=512, objective_resolution=8, objLabel= + protTilt = XmippProtTiltAnalysis(autoWindow=False, window_size=512, objective_resolution=8, objLabel= 'Tilt analysis window 512 obj resolution 8A') protTilt.inputMicrographs.set(self.protImport.outputMicrographs) self.proj.launchProtocol(protTilt, wait=True) @@ -1195,7 +1195,7 @@ def testTiltAnalysis(self): def testTiltAnalysis2(self): - protTilt2 = XmippProtTiltAnalysis(window_size=600, objective_resolution=4, + protTilt2 = XmippProtTiltAnalysis(autoWindow=False, window_size=600, objective_resolution=4, meanCorr_threshold=0.9, objLabel='Tilt analysis window 600 obj resolution 4A and threshold 0.9') protTilt2.inputMicrographs.set(self.protImport.outputMicrographs) From 7f5ccb510282ee7b24f6e054ce05027c1c808654 Mon Sep 17 00:00:00 2001 From: Daniel Marchan Date: Fri, 22 Mar 2024 12:08:11 +0000 Subject: [PATCH 12/12] minor sonar cloud issue --- xmipp3/protocols/protocol_tilt_analysis.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/xmipp3/protocols/protocol_tilt_analysis.py b/xmipp3/protocols/protocol_tilt_analysis.py index 3537f9d31..a4b13504e 100644 --- a/xmipp3/protocols/protocol_tilt_analysis.py +++ b/xmipp3/protocols/protocol_tilt_analysis.py @@ -345,9 +345,9 @@ def calculateTiltCorrelationStep(self, mic): micFolder = self._getOutputMicFolder(mic) micImage = ImageHandler().read(mic.getLocation()) dimx, dimy, z, n = micImage.getDimensions() - wind_step = self.windowSize - x_steps, y_steps = window_coordinates2D(dimx, dimy, wind_step) - subWindStep = int(wind_step * (self.samplingRate / self.objective_resolution.get())) + windStep = self.windowSize + x_steps, y_steps = window_coordinates2D(dimx, dimy, windStep) + subWindStep = int(windStep * (self.samplingRate / self.objective_resolution.get())) x_steps_psd, y_steps_psd = window_coordinates2D(subWindStep * 2, subWindStep * 2, subWindStep) # Extract windows window_image = ImageHandler().createImage() @@ -593,7 +593,7 @@ def rotation(imag, angle, shape, P): return transformed, M -def window_coordinates2D(x, y, wind_step): +def window_coordinates2D(x, y, windStep): x0 = 0 xF = x - 1 y0 = 0 @@ -601,15 +601,15 @@ def window_coordinates2D(x, y, wind_step): x_coor = [] y_coor = [] - if wind_step < x and wind_step < y: + if windStep < x and windStep < y: x_coor.append(x0) - x_coor.append(x0 + wind_step - 1) - x_coor.append(xF - wind_step) + x_coor.append(x0 + windStep - 1) + x_coor.append(xF - windStep) x_coor.append(xF) y_coor.append(y0) - y_coor.append(y0 + wind_step - 1) - y_coor.append(yF - wind_step) + y_coor.append(y0 + windStep - 1) + y_coor.append(yF - windStep) y_coor.append(yF) return x_coor, y_coor