From eb7a516c58bda9b48a4998ed725c8c929ec0935d Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 11 Jan 2023 10:55:06 +0100 Subject: [PATCH 001/104] Renamed conda env --- xmipp3/protocols/protocol_deep_hand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_deep_hand.py b/xmipp3/protocols/protocol_deep_hand.py index b784a779a..f86fc3ad3 100644 --- a/xmipp3/protocols/protocol_deep_hand.py +++ b/xmipp3/protocols/protocol_deep_hand.py @@ -39,7 +39,7 @@ class XmippProtDeepHand(EMProtocol, XmippProtocol): """ _label ="deep hand" - _conda_env = "xmipp_deepHand" + _conda_env = "xmipp_pytorch" def __init__(self, *args, **kwargs): EMProtocol.__init__(self, *args, **kwargs) From d283a96a94763207270124689ca1b152a29de842 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 20 Jan 2023 16:27:56 +0100 Subject: [PATCH 002/104] Initial commit --- xmipp3/protocols/__init__.py | 1 + xmipp3/protocols/protocol_reconstruct_fast.py | 171 ++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 xmipp3/protocols/protocol_reconstruct_fast.py diff --git a/xmipp3/protocols/__init__.py b/xmipp3/protocols/__init__.py index 1c9e07fa4..b93ab9ea2 100644 --- a/xmipp3/protocols/__init__.py +++ b/xmipp3/protocols/__init__.py @@ -140,3 +140,4 @@ from .protocol_volume_consensus import XmippProtVolConsensus from .protocol_classes_2d_mapping import XmippProtCL2DMap from .protocol_deep_hand import XmippProtDeepHand +from .protocol_reconstruct_fast import XmippProtReconstructFast diff --git a/xmipp3/protocols/protocol_reconstruct_fast.py b/xmipp3/protocols/protocol_reconstruct_fast.py new file mode 100644 index 000000000..084ae3a45 --- /dev/null +++ b/xmipp3/protocols/protocol_reconstruct_fast.py @@ -0,0 +1,171 @@ +# *************************************************************************** +# * Authors: Oier Lauzirika Zarrabeitia (oierlauzi@bizkaia.eu) +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 2 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'xmipp@cnb.csic.es' +# ***************************************************************************/ + +from pwem.protocols import ProtRefine3D +from pwem.objects import Volume, Transform, SetOfVolumes + +from pyworkflow.protocol.params import (Form, PointerParam, + FloatParam, IntParam, + StringParam, BooleanParam ) +from pyworkflow.utils.path import (cleanPath, makePath, copyFile, moveFile, + createLink, cleanPattern) + +import xmipp3 +from xmipp3.convert import writeSetOfParticles + +class XmippProtReconstructFast(ProtRefine3D, xmipp3.XmippProtocol): + _label = 'fast reconstruct' + _conda_env = 'xmipp_torch' + + def __init__(self, **kwargs): + ProtRefine3D.__init__(self, **kwargs) + + # --------------------------- DEFINE param functions -------------------------------------------- + def _defineParams(self, form: Form): + form.addSection(label='Input') + + form.addParam('inputParticles', PointerParam, label="Particles", important=True, + pointerClass='SetOfParticles') + form.addParam('inputVolume', PointerParam, label="Initial volumes", important=True, + pointerClass='Volume') + form.addParam('symmetryGroup', StringParam, default="c1", + label='Symmetry group', + help='If no symmetry is present, give c1') + form.addParam('correctCtf', BooleanParam, label="Correct CTF", default=True) + + form.addSection(label='Refinement') + form.addParam('resolutionLimit', FloatParam, label="Resolution limit (A)", default=10.0) + form.addParam('angularSampling', FloatParam, label="Angular sampling (º)", default=5.0) + form.addParam('shiftCount', IntParam, label="Shifts", default=9) + form.addParam('maxShift', FloatParam, label="Maximum shift (%)", default=10.0) + + form.addParallelSection(threads=1, mpi=8) + + def _insertAllSteps(self): + self._insertFunctionStep('convertInputStep') + self._insertFunctionStep('correctCtfStep') + self._insertFunctionStep('projectVolumeStep') + self._insertFunctionStep('trainDatabaseStep') + self._insertFunctionStep('alignStep') + self._insertFunctionStep('createOutputStep') + + def convertInputStep(self): + writeSetOfParticles(self.inputParticles.get(), + self._getInputParticleMdFilename()) + + def correctCtfStep(self): + particles = self.inputParticles.get() + + if self.correctCtf: + # Wiener filtering is required + args = [] + args += ['-i', self._getInputParticleMdFilename()] + args += ['-o', self._getWienerParticleStackFilename()] + args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] + args += ['--keep_input_columns'] + args += ['--sampling_rate', particles.getSamplingRate()] + + if (particles.isPhaseFlipped()): + args += ['--phase_flipped'] + + self.runJob('xmipp_ctf_correct_wiener2d', args) + + else: + # Wiener filtering is not requied, link the input md + createLink( + self._getInputParticleMdFilename(), + self._getWienerParticleMdFilename() + ) + + def projectVolumeStep(self): + args = [] + args += ['-i', self._getInputVolumeFilename()] + args += ['-o', self._getGalleryStackFilename()] + args += ['--sampling_rate', self.angularSampling] + args += ['--sym', self.symmetryGroup] + + self.runJob('xmipp_angular_project_library', args) + + def trainDatabaseStep(self): + expectedSize = int(2e6) # TODO determine form gallery + trainingSize = int(4e6) # TODO idem + + args = [] + args += ['-i', self._getGalleryMdFilename()] + args += ['-o', self._getTrainingIndexFilename()] + #args += ['--weights', self._getWeightsFilename()] + args += ['--max_shift', self._getMaxShift()] + args += ['--max_frequency', self._getDigitalFrequencyLimit()] + args += ['--method', 'fourier'] + args += ['--size', expectedSize] + args += ['--training', trainingSize] + + self.runJob('xmipp_train_database', args, numberOfMpi=1, env=self.getCondaEnv()) + + def alignStep(self): + args = [] + args += ['-i', self._getWienerParticleMdFilename()] + args += ['-o', self._getAlignmentMdFilename()] + #args += ['--weights', self._getWeightsFilename()] + args += ['--max_shift', self._getMaxShift()] + args += ['--max_frequency', self._getDigitalFrequencyLimit()] + args += ['--method', 'fourier'] + args += ['--size', expectedSize] + args += ['--training', trainingSize] + + self.runJob('xmipp_query_database', args, numberOfMpi=1, env=self.getCondaEnv()) + + def createOutputStep(self): + pass + + def _getDigitalFrequencyLimit(self): + return self.inputParticles.get().getSamplingRate() / float(self.resolutionLimit) + + def _getMaxShift(self): + return float(self.maxShift) / 100.0 + + def _getInputParticleMdFilename(self): + return self._getExtraPath('input_particles.xmd') + + def _getWienerParticleMdFilename(self): + return self._getExtraPath('input_particles_wiener.xmd') + + def _getWienerParticleStackFilename(self): + return self._getExtraPath('input_particles_wiener.mrcs') + + def _getInputVolumeFilename(self): + return self.inputVolume.get().getFileName() + + def _getGalleryMdFilename(self): + return self._getExtraPath('gallery.doc') + + def _getGalleryStackFilename(self): + return self._getExtraPath('gallery.mrcs') + + def _getWeightsFilename(self): + return self._getExtraPath('weights.mrc') + + def _getTrainingIndexFilename(self): + return self._getExtraPath('database.idx') + + def _getAlignmentMdFilename(self): + return self._getExtraPath('aligned.xmd') \ No newline at end of file From afe1f239938de9fbac383c9bddddde2581fda28f Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 25 Jan 2023 11:32:59 +0100 Subject: [PATCH 003/104] Continued working --- xmipp3/protocols/protocol_reconstruct_fast.py | 77 ++++++++++++++++--- 1 file changed, 66 insertions(+), 11 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_fast.py b/xmipp3/protocols/protocol_reconstruct_fast.py index 084ae3a45..0249457b0 100644 --- a/xmipp3/protocols/protocol_reconstruct_fast.py +++ b/xmipp3/protocols/protocol_reconstruct_fast.py @@ -25,7 +25,8 @@ from pyworkflow.protocol.params import (Form, PointerParam, FloatParam, IntParam, - StringParam, BooleanParam ) + StringParam, BooleanParam, + LEVEL_ADVANCED, USE_GPU, GPU_LIST ) from pyworkflow.utils.path import (cleanPath, makePath, copyFile, moveFile, createLink, cleanPattern) @@ -41,6 +42,16 @@ def __init__(self, **kwargs): # --------------------------- DEFINE param functions -------------------------------------------- def _defineParams(self, form: Form): + form.addHidden(USE_GPU, BooleanParam, default=True, + label="Use GPU for execution", + help="This protocol has both CPU and GPU implementation.\ + Select the one you want to use.") + + form.addHidden(GPU_LIST, StringParam, default='0', + expertLevel=LEVEL_ADVANCED, + label="Choose GPU IDs", + help="Add a list of GPU devices that can be used") + form.addSection(label='Input') form.addParam('inputParticles', PointerParam, label="Particles", important=True, @@ -60,14 +71,26 @@ def _defineParams(self, form: Form): form.addParallelSection(threads=1, mpi=8) + #--------------------------- INFO functions -------------------------------------------- + + + #--------------------------- INSERT steps functions -------------------------------------------- def _insertAllSteps(self): - self._insertFunctionStep('convertInputStep') - self._insertFunctionStep('correctCtfStep') - self._insertFunctionStep('projectVolumeStep') - self._insertFunctionStep('trainDatabaseStep') - self._insertFunctionStep('alignStep') - self._insertFunctionStep('createOutputStep') + convertInputStepId = self._insertFunctionStep('convertInputStep', prerequisites=[]) + correctCtfStepId = self._insertFunctionStep('correctCtfStep', prerequisites=[convertInputStepId]) + projectVolumeStepId = self._insertFunctionStep('projectVolumeStep', prerequisites=[convertInputStepId]) + trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', prerequisites=[projectVolumeStepId]) + alignStepId = self._insertFunctionStep('alignStep', prerequisites=[correctCtfStepId, trainDatabaseStepId]) + reconstructIds = self._insertReconstructSteps(prerequisites=[alignStepId]) + createOutputStepId = self._insertFunctionStep('createOutputStep', prerequisites=reconstructIds) + def _insertReconstructSteps(self, prerequisites): + splitStepId = self._insertFunctionStep('splitStep', prerequisites=prerequisites) + reconstructStepId1 = self._insertFunctionStep('reconstructStep', i=1, prerequisites=[splitStepId]) + reconstructStepId2 = self._insertFunctionStep('reconstructStep', i=2, prerequisites=[splitStepId]) + return [reconstructStepId1, reconstructStepId2] + + #--------------------------- STEPS functions -------------------------------------------- def convertInputStep(self): writeSetOfParticles(self.inputParticles.get(), self._getInputParticleMdFilename()) @@ -90,7 +113,7 @@ def correctCtfStep(self): self.runJob('xmipp_ctf_correct_wiener2d', args) else: - # Wiener filtering is not requied, link the input md + # Wiener filtering is not required, link the input md createLink( self._getInputParticleMdFilename(), self._getWienerParticleMdFilename() @@ -118,25 +141,51 @@ def trainDatabaseStep(self): args += ['--method', 'fourier'] args += ['--size', expectedSize] args += ['--training', trainingSize] + if self.useGpu: + args += ['--gpu', 0] # TODO select self.runJob('xmipp_train_database', args, numberOfMpi=1, env=self.getCondaEnv()) def alignStep(self): + nRotations = round(360 / self.angularSampling) + nShift = self.shiftCount + args = [] args += ['-i', self._getWienerParticleMdFilename()] args += ['-o', self._getAlignmentMdFilename()] + args += ['--index', self._getTrainingIndexFilename()] #args += ['--weights', self._getWeightsFilename()] args += ['--max_shift', self._getMaxShift()] + args += ['--rotations', nRotations] + args += ['--shifts', nShift] args += ['--max_frequency', self._getDigitalFrequencyLimit()] args += ['--method', 'fourier'] - args += ['--size', expectedSize] - args += ['--training', trainingSize] + args += ['--dropna'] + if self.useGpu: + args += ['--gpu', 0] # TODO select self.runJob('xmipp_query_database', args, numberOfMpi=1, env=self.getCondaEnv()) + def splitStep(self): + args = [] + args += ['-i', self._getAlignmentMdFilename()] + args += ['-n', 2] + + self.runJob('xmipp_metadata_split', args, numberOfMpi=1) + + def reconstructStep(self, i: int): + args [] + args += ['-i', self._getAlignmentHalfMdFilename(i)] + args += ['-o', self._getHalfVolumeFilename(i)] + args += ['--sym', self.symmetryGroup.get()] + args += ['--weight'] + + self.runJob("xmipp_reconstruct_fourier_accel", args) + def createOutputStep(self): pass + #--------------------------- UTILS functions -------------------------------------------- def _getDigitalFrequencyLimit(self): return self.inputParticles.get().getSamplingRate() / float(self.resolutionLimit) @@ -168,4 +217,10 @@ def _getTrainingIndexFilename(self): return self._getExtraPath('database.idx') def _getAlignmentMdFilename(self): - return self._getExtraPath('aligned.xmd') \ No newline at end of file + return self._getExtraPath('aligned.xmd') + + def _getAlignmentHalfMdFilename(self, i: int): + return self._getExtraPath('aligned%06d.xmd' % i) + + def _getHalfVolumeFilename(self, i: int): + return self._getExtraPath('volume%01d.mrc' % i) \ No newline at end of file From 1d62216012ca90a06ec904d95d08cc5faf29616d Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 26 Jan 2023 13:04:35 +0100 Subject: [PATCH 004/104] Continued working on the protocol --- xmipp3/protocols/protocol_reconstruct_fast.py | 103 ++++++++++++------ 1 file changed, 70 insertions(+), 33 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_fast.py b/xmipp3/protocols/protocol_reconstruct_fast.py index 0249457b0..6cefc0a4e 100644 --- a/xmipp3/protocols/protocol_reconstruct_fast.py +++ b/xmipp3/protocols/protocol_reconstruct_fast.py @@ -22,6 +22,7 @@ from pwem.protocols import ProtRefine3D from pwem.objects import Volume, Transform, SetOfVolumes +from pwem import Config from pyworkflow.protocol.params import (Form, PointerParam, FloatParam, IntParam, @@ -61,7 +62,6 @@ def _defineParams(self, form: Form): form.addParam('symmetryGroup', StringParam, default="c1", label='Symmetry group', help='If no symmetry is present, give c1') - form.addParam('correctCtf', BooleanParam, label="Correct CTF", default=True) form.addSection(label='Refinement') form.addParam('resolutionLimit', FloatParam, label="Resolution limit (A)", default=10.0) @@ -81,15 +81,17 @@ def _insertAllSteps(self): projectVolumeStepId = self._insertFunctionStep('projectVolumeStep', prerequisites=[convertInputStepId]) trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', prerequisites=[projectVolumeStepId]) alignStepId = self._insertFunctionStep('alignStep', prerequisites=[correctCtfStepId, trainDatabaseStepId]) - reconstructIds = self._insertReconstructSteps(prerequisites=[alignStepId]) - createOutputStepId = self._insertFunctionStep('createOutputStep', prerequisites=reconstructIds) + reconstructId = self._insertReconstructSteps(prerequisites=[alignStepId]) + createOutputStepId = self._insertFunctionStep('createOutputStep', prerequisites=reconstructId) def _insertReconstructSteps(self, prerequisites): splitStepId = self._insertFunctionStep('splitStep', prerequisites=prerequisites) - reconstructStepId1 = self._insertFunctionStep('reconstructStep', i=1, prerequisites=[splitStepId]) - reconstructStepId2 = self._insertFunctionStep('reconstructStep', i=2, prerequisites=[splitStepId]) - return [reconstructStepId1, reconstructStepId2] - + reconstructStepId1 = self._insertFunctionStep('reconstructStep', 1, prerequisites=[splitStepId]) + reconstructStepId2 = self._insertFunctionStep('reconstructStep', 2, prerequisites=[splitStepId]) + computeFscStepId = self._insertFunctionStep('computeFscStep', prerequisites=[reconstructStepId1, reconstructStepId2]) + averageVolumeStepId = self._insertFunctionStep('averageVolumeStep', prerequisites=[reconstructStepId1, reconstructStepId2]) + return [computeFscStepId, averageVolumeStepId] + #--------------------------- STEPS functions -------------------------------------------- def convertInputStep(self): writeSetOfParticles(self.inputParticles.get(), @@ -98,26 +100,16 @@ def convertInputStep(self): def correctCtfStep(self): particles = self.inputParticles.get() - if self.correctCtf: - # Wiener filtering is required - args = [] - args += ['-i', self._getInputParticleMdFilename()] - args += ['-o', self._getWienerParticleStackFilename()] - args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] - args += ['--keep_input_columns'] - args += ['--sampling_rate', particles.getSamplingRate()] - - if (particles.isPhaseFlipped()): - args += ['--phase_flipped'] + args = [] + args += ['-i', self._getInputParticleMdFilename()] + args += ['-o', self._getWienerParticleStackFilename()] + args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] + args += ['--keep_input_columns'] + args += ['--sampling_rate', self._getSamplingRate()] + if (particles.isPhaseFlipped()): + args += ['--phase_flipped'] - self.runJob('xmipp_ctf_correct_wiener2d', args) - - else: - # Wiener filtering is not required, link the input md - createLink( - self._getInputParticleMdFilename(), - self._getWienerParticleMdFilename() - ) + self.runJob('xmipp_ctf_correct_wiener2d', args) def projectVolumeStep(self): args = [] @@ -144,15 +136,19 @@ def trainDatabaseStep(self): if self.useGpu: args += ['--gpu', 0] # TODO select - self.runJob('xmipp_train_database', args, numberOfMpi=1, env=self.getCondaEnv()) + env = self.getCondaEnv() + env['LD_LIBRARY_PATH'] = '' # Torch does not like it + self.runJob('xmipp_train_database', args, numberOfMpi=1, env=env) def alignStep(self): - nRotations = round(360 / self.angularSampling) + batchSize = 1024 + nRotations = round(360 / float(self.angularSampling)) nShift = self.shiftCount args = [] args += ['-i', self._getWienerParticleMdFilename()] args += ['-o', self._getAlignmentMdFilename()] + args += ['-r', self._getGalleryMdFilename()] args += ['--index', self._getTrainingIndexFilename()] #args += ['--weights', self._getWeightsFilename()] args += ['--max_shift', self._getMaxShift()] @@ -160,11 +156,14 @@ def alignStep(self): args += ['--shifts', nShift] args += ['--max_frequency', self._getDigitalFrequencyLimit()] args += ['--method', 'fourier'] - args += ['--dropna'] + #args += ['--dropna'] + args += ['--batch', batchSize] if self.useGpu: args += ['--gpu', 0] # TODO select - self.runJob('xmipp_query_database', args, numberOfMpi=1, env=self.getCondaEnv()) + env = self.getCondaEnv() + env['LD_LIBRARY_PATH'] = '' # Torch does not like it + self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) def splitStep(self): args = [] @@ -174,16 +173,45 @@ def splitStep(self): self.runJob('xmipp_metadata_split', args, numberOfMpi=1) def reconstructStep(self, i: int): - args [] + args = [] args += ['-i', self._getAlignmentHalfMdFilename(i)] args += ['-o', self._getHalfVolumeFilename(i)] args += ['--sym', self.symmetryGroup.get()] args += ['--weight'] self.runJob("xmipp_reconstruct_fourier_accel", args) + + def computeFscStep(self): + args = [] + args += ['-r', self._getHalfVolumeFilename(1)] + args += ['-i', self._getHalfVolumeFilename(2)] + args += ['-o', self._getFscFilename()] + args += ['--sampling_rate', self._getSamplingRate()] + + self.runJob('xmipp_resolution_fsc', args, numberOfMpi=1) + + def averageVolumeStep(self): + args = [] + args += ['-i', self._getAlignmentHalfMdFilename(1)] + args += ['--plus', self._getAlignmentHalfMdFilename(2)] + args += ['-o', self._getAverageVolumeFilename()] + self.runJob('xmipp_image_operate', args, numberOfMpi=1) + + args = [] + args += ['-i', self._getAverageVolumeFilename()] + args += ['--mult 0.5'] + self.runJob('xmipp_image_operate', args, numberOfMpi=1) def createOutputStep(self): - pass + volume=Volume() + volume.setFileName(self._getAverageVolumeFilename()) + volume.setSamplingRate(self._getSamplingRate()) + volume.setHalfMaps([ + self._getHalfVolumeFilename(1), + self._getHalfVolumeFilename(2) + ]) + self._defineOutputs(outputVolume=volume) + self._defineSourceRelation(self.inputParticles.get(), volume) #--------------------------- UTILS functions -------------------------------------------- def _getDigitalFrequencyLimit(self): @@ -192,6 +220,9 @@ def _getDigitalFrequencyLimit(self): def _getMaxShift(self): return float(self.maxShift) / 100.0 + def _getSamplingRate(self): + return self.inputParticles.get().getSamplingRate() + def _getInputParticleMdFilename(self): return self._getExtraPath('input_particles.xmd') @@ -223,4 +254,10 @@ def _getAlignmentHalfMdFilename(self, i: int): return self._getExtraPath('aligned%06d.xmd' % i) def _getHalfVolumeFilename(self, i: int): - return self._getExtraPath('volume%01d.mrc' % i) \ No newline at end of file + return self._getExtraPath('volume%01d.mrc' % i) + + def _getAverageVolumeFilename(self): + return self._getExtraPath('average_volume.mrc') + + def _getFscFilename(self): + return self._getExtraPath('fsc.xmd') \ No newline at end of file From 23e3a59eb27db09af6dcfcadd6125d4b7e788a99 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 27 Jan 2023 10:52:36 +0100 Subject: [PATCH 005/104] Updated output generation --- xmipp3/protocols/protocol_reconstruct_fast.py | 81 ++++++++++++++----- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_fast.py b/xmipp3/protocols/protocol_reconstruct_fast.py index 6cefc0a4e..28c7d97a8 100644 --- a/xmipp3/protocols/protocol_reconstruct_fast.py +++ b/xmipp3/protocols/protocol_reconstruct_fast.py @@ -21,8 +21,8 @@ # ***************************************************************************/ from pwem.protocols import ProtRefine3D -from pwem.objects import Volume, Transform, SetOfVolumes -from pwem import Config +from pwem.objects import Volume, FSC +from pwem import emlib from pyworkflow.protocol.params import (Form, PointerParam, FloatParam, IntParam, @@ -32,7 +32,7 @@ createLink, cleanPattern) import xmipp3 -from xmipp3.convert import writeSetOfParticles +from xmipp3.convert import writeSetOfParticles, readSetOfParticles class XmippProtReconstructFast(ProtRefine3D, xmipp3.XmippProtocol): _label = 'fast reconstruct' @@ -106,6 +106,8 @@ def correctCtfStep(self): args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] args += ['--keep_input_columns'] args += ['--sampling_rate', self._getSamplingRate()] + args += ['--pad', '2'] + args += ['--wc', '-1.0'] if (particles.isPhaseFlipped()): args += ['--phase_flipped'] @@ -156,7 +158,7 @@ def alignStep(self): args += ['--shifts', nShift] args += ['--max_frequency', self._getDigitalFrequencyLimit()] args += ['--method', 'fourier'] - #args += ['--dropna'] + args += ['--dropna'] args += ['--batch', batchSize] if self.useGpu: args += ['--gpu', 0] # TODO select @@ -183,7 +185,7 @@ def reconstructStep(self, i: int): def computeFscStep(self): args = [] - args += ['-r', self._getHalfVolumeFilename(1)] + args += ['--ref', self._getHalfVolumeFilename(1)] args += ['-i', self._getHalfVolumeFilename(2)] args += ['-o', self._getFscFilename()] args += ['--sampling_rate', self._getSamplingRate()] @@ -192,26 +194,21 @@ def computeFscStep(self): def averageVolumeStep(self): args = [] - args += ['-i', self._getAlignmentHalfMdFilename(1)] - args += ['--plus', self._getAlignmentHalfMdFilename(2)] + args += ['-i', self._getHalfVolumeFilename(1)] + args += ['--plus', self._getHalfVolumeFilename(2)] args += ['-o', self._getAverageVolumeFilename()] self.runJob('xmipp_image_operate', args, numberOfMpi=1) args = [] args += ['-i', self._getAverageVolumeFilename()] - args += ['--mult 0.5'] + args += ['--mult', '0.5'] self.runJob('xmipp_image_operate', args, numberOfMpi=1) def createOutputStep(self): - volume=Volume() - volume.setFileName(self._getAverageVolumeFilename()) - volume.setSamplingRate(self._getSamplingRate()) - volume.setHalfMaps([ - self._getHalfVolumeFilename(1), - self._getHalfVolumeFilename(2) - ]) - self._defineOutputs(outputVolume=volume) - self._defineSourceRelation(self.inputParticles.get(), volume) + self._createOutputParticleSet() + self._createOutputVolume() + self._createOutputFsc() + #--------------------------- UTILS functions -------------------------------------------- def _getDigitalFrequencyLimit(self): @@ -260,4 +257,52 @@ def _getAverageVolumeFilename(self): return self._getExtraPath('average_volume.mrc') def _getFscFilename(self): - return self._getExtraPath('fsc.xmd') \ No newline at end of file + return self._getExtraPath('fsc.xmd') + + def _createOutputVolume(self): + volume=Volume() + + # Fill + volume.setFileName(self._getAverageVolumeFilename()) + volume.setSamplingRate(self._getSamplingRate()) + volume.setHalfMaps([ + self._getHalfVolumeFilename(1), + self._getHalfVolumeFilename(2) + ]) + + # Define the output + self._defineOutputs(outputVolume=volume) + self._defineSourceRelation(self.inputParticles.get(), volume) + + return volume + + def _createOutputParticleSet(self): + particleSet = self._createSetOfParticles() + + # TODO replace wiener corrected images + + # Fill + readSetOfParticles(self._getAlignmentMdFilename(), particleSet) + + # Define the output + self._defineOutputs(outputParticles=particleSet) + self._defineSourceRelation(self.inputParticles.get(), particleSet) + + return particleSet + + def _createOutputFsc(self): + fsc = FSC() + + # Load from metadata + fsc.loadFromMd( + self._getFscFilename(), + emlib.MDL_RESOLUTION_FREQ, + emlib.MDL_RESOLUTION_FRC + ) + + # Define the output + self._defineOutputs(outputFSC=fsc) + self._defineSourceRelation(self.inputParticles.get(), fsc) + + return fsc + \ No newline at end of file From 7cbaa7e2c66c442588b37df498f1d604a567dec2 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 27 Jan 2023 16:07:10 +0100 Subject: [PATCH 006/104] Modified training dataset size --- xmipp3/protocols/protocol_reconstruct_fast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_fast.py b/xmipp3/protocols/protocol_reconstruct_fast.py index 28c7d97a8..02878c7de 100644 --- a/xmipp3/protocols/protocol_reconstruct_fast.py +++ b/xmipp3/protocols/protocol_reconstruct_fast.py @@ -124,7 +124,7 @@ def projectVolumeStep(self): def trainDatabaseStep(self): expectedSize = int(2e6) # TODO determine form gallery - trainingSize = int(4e6) # TODO idem + trainingSize = int(2e6) # TODO idem args = [] args += ['-i', self._getGalleryMdFilename()] From ba5ae311ca04df48733cf338fd44c9f19caecf5e Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 27 Jan 2023 16:36:39 +0100 Subject: [PATCH 007/104] Added sampling rate to output --- xmipp3/protocols/protocol_reconstruct_fast.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xmipp3/protocols/protocol_reconstruct_fast.py b/xmipp3/protocols/protocol_reconstruct_fast.py index 02878c7de..1495c4bbf 100644 --- a/xmipp3/protocols/protocol_reconstruct_fast.py +++ b/xmipp3/protocols/protocol_reconstruct_fast.py @@ -283,6 +283,7 @@ def _createOutputParticleSet(self): # Fill readSetOfParticles(self._getAlignmentMdFilename(), particleSet) + particleSet.setSamplingRate(self._getSamplingRate()) # Define the output self._defineOutputs(outputParticles=particleSet) From 9054c2f45d524b93c3beacad625d523ba4bb1bf0 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 30 Jan 2023 10:23:25 +0100 Subject: [PATCH 008/104] Renamed protocol to swiftres --- xmipp3/protocols/__init__.py | 2 +- ...l_reconstruct_fast.py => protocol_reconstruct_swiftres.py} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename xmipp3/protocols/{protocol_reconstruct_fast.py => protocol_reconstruct_swiftres.py} (99%) diff --git a/xmipp3/protocols/__init__.py b/xmipp3/protocols/__init__.py index b93ab9ea2..788e641cc 100644 --- a/xmipp3/protocols/__init__.py +++ b/xmipp3/protocols/__init__.py @@ -140,4 +140,4 @@ from .protocol_volume_consensus import XmippProtVolConsensus from .protocol_classes_2d_mapping import XmippProtCL2DMap from .protocol_deep_hand import XmippProtDeepHand -from .protocol_reconstruct_fast import XmippProtReconstructFast +from .protocol_reconstruct_swiftres import XmippProtReconstructSwiftres diff --git a/xmipp3/protocols/protocol_reconstruct_fast.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py similarity index 99% rename from xmipp3/protocols/protocol_reconstruct_fast.py rename to xmipp3/protocols/protocol_reconstruct_swiftres.py index 1495c4bbf..a7efa740d 100644 --- a/xmipp3/protocols/protocol_reconstruct_fast.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -34,8 +34,8 @@ import xmipp3 from xmipp3.convert import writeSetOfParticles, readSetOfParticles -class XmippProtReconstructFast(ProtRefine3D, xmipp3.XmippProtocol): - _label = 'fast reconstruct' +class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): + _label = 'swiftres' _conda_env = 'xmipp_torch' def __init__(self, **kwargs): From dbf20f74d918cc4bd3f84f4a581612132ebb5dff Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 30 Jan 2023 10:52:28 +0100 Subject: [PATCH 009/104] Added GPU support to fourier reconstruction --- .../protocol_reconstruct_swiftres.py | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index a7efa740d..5d79abd15 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -181,7 +181,46 @@ def reconstructStep(self, i: int): args += ['--sym', self.symmetryGroup.get()] args += ['--weight'] - self.runJob("xmipp_reconstruct_fourier_accel", args) + # Determine the execution parameters + numberOfMpi = self.numberOfMpi.get() + if self.useGpu.get(): + reconstructProgram = 'xmipp_cuda_reconstruct_fourier' + + gpuList = self.getGpuList() + if self.numberOfMpi.get() > 1: + numberOfGpus = len(gpuList) + numberOfMpi = numberOfGpus + 1 + args += ['-gpusPerNode', numberOfGpus] + args += ['-threadsPerGPU', max(self.numberOfThreads.get(), 4)] + else: + args += ['--device', ','.join(gpuList)] + + """ + count=0 + GpuListCuda='' + if self.useQueueForSteps() or self.useQueue(): + GpuList = os.environ["CUDA_VISIBLE_DEVICES"] + GpuList = GpuList.split(",") + for elem in GpuList: + GpuListCuda = GpuListCuda+str(count)+' ' + count+=1 + else: + GpuListAux = '' + for elem in self.getGpuList(): + GpuListCuda = GpuListCuda+str(count)+' ' + GpuListAux = GpuListAux+str(elem)+',' + count+=1 + os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux + """ + + args += ['--thr', self.numberOfThreads.get()] + + else: + reconstructProgram = 'xmipp_reconstruct_fourier_accel' + + # Run + self.runJob(reconstructProgram, args, numberOfMpi=numberOfMpi) + def computeFscStep(self): args = [] From 87d3c73858382cf8231b3203c0269df375cdaf09 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 30 Jan 2023 12:07:47 +0100 Subject: [PATCH 010/104] Added iteration structure --- .../protocol_reconstruct_swiftres.py | 181 ++++++++++++------ 1 file changed, 121 insertions(+), 60 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 5d79abd15..e0a1a014f 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -78,20 +78,38 @@ def _defineParams(self, form: Form): def _insertAllSteps(self): convertInputStepId = self._insertFunctionStep('convertInputStep', prerequisites=[]) correctCtfStepId = self._insertFunctionStep('correctCtfStep', prerequisites=[convertInputStepId]) - projectVolumeStepId = self._insertFunctionStep('projectVolumeStep', prerequisites=[convertInputStepId]) - trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', prerequisites=[projectVolumeStepId]) - alignStepId = self._insertFunctionStep('alignStep', prerequisites=[correctCtfStepId, trainDatabaseStepId]) - reconstructId = self._insertReconstructSteps(prerequisites=[alignStepId]) - createOutputStepId = self._insertFunctionStep('createOutputStep', prerequisites=reconstructId) + + lastIds = [correctCtfStepId] + for i in range(1): # TODO + lastIds = self._insertIterationSteps(i, prerequisites=lastIds) + + self._insertFunctionStep('createOutputStep', prerequisites=lastIds) - def _insertReconstructSteps(self, prerequisites): - splitStepId = self._insertFunctionStep('splitStep', prerequisites=prerequisites) - reconstructStepId1 = self._insertFunctionStep('reconstructStep', 1, prerequisites=[splitStepId]) - reconstructStepId2 = self._insertFunctionStep('reconstructStep', 2, prerequisites=[splitStepId]) - computeFscStepId = self._insertFunctionStep('computeFscStep', prerequisites=[reconstructStepId1, reconstructStepId2]) - averageVolumeStepId = self._insertFunctionStep('averageVolumeStep', prerequisites=[reconstructStepId1, reconstructStepId2]) + def _insertIterationSteps(self, iteration: int, prerequisites): + setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, prerequisites=prerequisites) + alignIds = self._insertAlignmentSteps(iteration, prerequisites=[setupIterationStepId]) + reconstructIds = self._insertReconstructSteps(iteration, prerequisites=alignIds) + postProcessIds = self._insertPostProcessSteps(iteration, prerequisites=reconstructIds) + return postProcessIds + + def _insertAlignmentSteps(self, iteration: int, prerequisites): + projectVolumeStepId = self._insertFunctionStep('projectVolumeStep', iteration, prerequisites=prerequisites) + trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=[projectVolumeStepId]) + alignStepId = self._insertFunctionStep('alignStep', iteration, prerequisites=[trainDatabaseStepId]) + return [alignStepId] + + def _insertReconstructSteps(self, iteration: int, prerequisites): + splitStepId = self._insertFunctionStep('splitStep', iteration, prerequisites=prerequisites) + reconstructStepId1 = self._insertFunctionStep('reconstructStep', iteration, 1, prerequisites=[splitStepId]) + reconstructStepId2 = self._insertFunctionStep('reconstructStep', iteration, 2, prerequisites=[splitStepId]) + computeFscStepId = self._insertFunctionStep('computeFscStep', iteration, prerequisites=[reconstructStepId1, reconstructStepId2]) + averageVolumeStepId = self._insertFunctionStep('averageVolumeStep', iteration, prerequisites=[reconstructStepId1, reconstructStepId2]) return [computeFscStepId, averageVolumeStepId] + def _insertPostProcessSteps(self, iteration: int, prerequisites): + filterVolumeStepId = self._insertFunctionStep('filterVolumeStep', iteration, prerequisites=prerequisites) + return [filterVolumeStepId] + #--------------------------- STEPS functions -------------------------------------------- def convertInputStep(self): writeSetOfParticles(self.inputParticles.get(), @@ -113,23 +131,26 @@ def correctCtfStep(self): self.runJob('xmipp_ctf_correct_wiener2d', args) - def projectVolumeStep(self): + def setupIterationStep(self, iteration: int): + makePath(self._getIterationPath(iteration)) + + def projectVolumeStep(self, iteration: int): args = [] - args += ['-i', self._getInputVolumeFilename()] - args += ['-o', self._getGalleryStackFilename()] + args += ['-i', self._getIterationInputVolumeFilename(iteration)] + args += ['-o', self._getGalleryStackFilename(iteration)] args += ['--sampling_rate', self.angularSampling] args += ['--sym', self.symmetryGroup] self.runJob('xmipp_angular_project_library', args) - def trainDatabaseStep(self): + def trainDatabaseStep(self, iteration: int): expectedSize = int(2e6) # TODO determine form gallery trainingSize = int(2e6) # TODO idem args = [] - args += ['-i', self._getGalleryMdFilename()] - args += ['-o', self._getTrainingIndexFilename()] - #args += ['--weights', self._getWeightsFilename()] + args += ['-i', self._getGalleryMdFilename(iteration)] + args += ['-o', self._getTrainingIndexFilename(iteration)] + #args += ['--weights', self._getWeightsFilename(iteration)] args += ['--max_shift', self._getMaxShift()] args += ['--max_frequency', self._getDigitalFrequencyLimit()] args += ['--method', 'fourier'] @@ -142,17 +163,17 @@ def trainDatabaseStep(self): env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_train_database', args, numberOfMpi=1, env=env) - def alignStep(self): + def alignStep(self, iteration: int): batchSize = 1024 nRotations = round(360 / float(self.angularSampling)) nShift = self.shiftCount args = [] args += ['-i', self._getWienerParticleMdFilename()] - args += ['-o', self._getAlignmentMdFilename()] - args += ['-r', self._getGalleryMdFilename()] - args += ['--index', self._getTrainingIndexFilename()] - #args += ['--weights', self._getWeightsFilename()] + args += ['-o', self._getAlignmentMdFilename(iteration)] + args += ['-r', self._getGalleryMdFilename(iteration)] + args += ['--index', self._getTrainingIndexFilename(iteration)] + #args += ['--weights', self._getWeightsFilename(iteration)] args += ['--max_shift', self._getMaxShift()] args += ['--rotations', nRotations] args += ['--shifts', nShift] @@ -167,17 +188,17 @@ def alignStep(self): env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) - def splitStep(self): + def splitStep(self, iteration: int): args = [] - args += ['-i', self._getAlignmentMdFilename()] + args += ['-i', self._getAlignmentMdFilename(iteration)] args += ['-n', 2] self.runJob('xmipp_metadata_split', args, numberOfMpi=1) - def reconstructStep(self, i: int): + def reconstructStep(self, iteration: int, half: int): args = [] - args += ['-i', self._getAlignmentHalfMdFilename(i)] - args += ['-o', self._getHalfVolumeFilename(i)] + args += ['-i', self._getAlignmentHalfMdFilename(iteration, half)] + args += ['-o', self._getHalfVolumeFilename(iteration, half)] args += ['--sym', self.symmetryGroup.get()] args += ['--weight'] @@ -222,27 +243,39 @@ def reconstructStep(self, i: int): self.runJob(reconstructProgram, args, numberOfMpi=numberOfMpi) - def computeFscStep(self): + def computeFscStep(self, iteration: int): args = [] - args += ['--ref', self._getHalfVolumeFilename(1)] - args += ['-i', self._getHalfVolumeFilename(2)] - args += ['-o', self._getFscFilename()] + args += ['--ref', self._getHalfVolumeFilename(iteration, 1)] + args += ['-i', self._getHalfVolumeFilename(iteration, 2)] + args += ['-o', self._getFscFilename(iteration)] args += ['--sampling_rate', self._getSamplingRate()] self.runJob('xmipp_resolution_fsc', args, numberOfMpi=1) - def averageVolumeStep(self): + def averageVolumeStep(self, iteration: int): args = [] - args += ['-i', self._getHalfVolumeFilename(1)] - args += ['--plus', self._getHalfVolumeFilename(2)] - args += ['-o', self._getAverageVolumeFilename()] + args += ['-i', self._getHalfVolumeFilename(iteration, 1)] + args += ['--plus', self._getHalfVolumeFilename(iteration, 2)] + args += ['-o', self._getAverageVolumeFilename(iteration)] self.runJob('xmipp_image_operate', args, numberOfMpi=1) args = [] - args += ['-i', self._getAverageVolumeFilename()] + args += ['-i', self._getAverageVolumeFilename(iteration)] args += ['--mult', '0.5'] self.runJob('xmipp_image_operate', args, numberOfMpi=1) + def filterVolumeStep(self, iteration: int): + mdFsc = emlib.MetaData(self._getFscFilename(iteration)) + resolution = self._computeResolution(mdFsc, self._getSamplingRate(), 0.5) + + args = [] + args += ['-i', self._getAverageVolumeFilename(iteration)] + args += ['-o', self._getFilteredVolumeFilename(iteration)] + args += ['--fourier', 'low_pass', resolution] + args += ['--sampling', self._getSamplingRate()] + + self.runJob('xmipp_transform_filter', args, numberOfMpi=1) + def createOutputStep(self): self._createOutputParticleSet() self._createOutputVolume() @@ -259,6 +292,9 @@ def _getMaxShift(self): def _getSamplingRate(self): return self.inputParticles.get().getSamplingRate() + def _getIterationPath(self, iteration: int, *paths): + return self._getExtraPath('iteration_%04d' % iteration, *paths) + def _getInputParticleMdFilename(self): return self._getExtraPath('input_particles.xmd') @@ -271,42 +307,65 @@ def _getWienerParticleStackFilename(self): def _getInputVolumeFilename(self): return self.inputVolume.get().getFileName() - def _getGalleryMdFilename(self): - return self._getExtraPath('gallery.doc') + def _getIterationInputVolumeFilename(self, iteration: int): + if iteration == 0: + return self._getInputVolumeFilename() + else: + return self._getAverageVolumeFilename(iteration-1) + + def _getGalleryMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'gallery.doc') - def _getGalleryStackFilename(self): - return self._getExtraPath('gallery.mrcs') + def _getGalleryStackFilename(self, iteration: int): + return self._getIterationPath(iteration, 'gallery.mrcs') - def _getWeightsFilename(self): - return self._getExtraPath('weights.mrc') + def _getWeightsFilename(self, iteration: int): + return self._getIterationPath(iteration, 'weights.mrc') - def _getTrainingIndexFilename(self): - return self._getExtraPath('database.idx') + def _getTrainingIndexFilename(self, iteration: int): + return self._getIterationPath(iteration, 'database.idx') - def _getAlignmentMdFilename(self): - return self._getExtraPath('aligned.xmd') + def _getAlignmentMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'aligned.xmd') - def _getAlignmentHalfMdFilename(self, i: int): - return self._getExtraPath('aligned%06d.xmd' % i) + def _getAlignmentHalfMdFilename(self, iteration: int, half: int): + return self._getIterationPath(iteration, 'aligned%06d.xmd' % half) - def _getHalfVolumeFilename(self, i: int): - return self._getExtraPath('volume%01d.mrc' % i) + def _getHalfVolumeFilename(self, iteration: int, half: int): + return self._getIterationPath(iteration, 'volume_half%01d.mrc' % half) - def _getAverageVolumeFilename(self): - return self._getExtraPath('average_volume.mrc') + def _getAverageVolumeFilename(self, iteration: int): + return self._getIterationPath(iteration, 'volume_avg.mrc') - def _getFscFilename(self): - return self._getExtraPath('fsc.xmd') + def _getFscFilename(self, iteration: int): + return self._getIterationPath(iteration, 'fsc.xmd') + + def _getFilteredVolumeFilename(self, iteration: int): + return self._getIterationPath(iteration, 'volume_filtered.mrc') + + + def _computeResolution(self, mdFsc, Ts, threshold): + resolution = 2 * Ts + + # Iterate until the FSC is under the threshold + for objId in mdFsc: + fsc = mdFsc.getValue(emlib.MDL_RESOLUTION_FRC, objId) + if fsc < threshold: + resolution = mdFsc.getValue(emlib.MDL_RESOLUTION_FREQREAL, objId) + break + + return resolution def _createOutputVolume(self): volume=Volume() # Fill - volume.setFileName(self._getAverageVolumeFilename()) + lastIteration = 0 # TODO + volume.setFileName(self._getAverageVolumeFilename(lastIteration)) volume.setSamplingRate(self._getSamplingRate()) volume.setHalfMaps([ - self._getHalfVolumeFilename(1), - self._getHalfVolumeFilename(2) + self._getHalfVolumeFilename(lastIteration, 1), + self._getHalfVolumeFilename(lastIteration, 2) ]) # Define the output @@ -321,7 +380,8 @@ def _createOutputParticleSet(self): # TODO replace wiener corrected images # Fill - readSetOfParticles(self._getAlignmentMdFilename(), particleSet) + lastIteration = 0 # TODO + readSetOfParticles(self._getAlignmentMdFilename(lastIteration), particleSet) particleSet.setSamplingRate(self._getSamplingRate()) # Define the output @@ -334,8 +394,9 @@ def _createOutputFsc(self): fsc = FSC() # Load from metadata + lastIteration = 0 # TODO fsc.loadFromMd( - self._getFscFilename(), + self._getFscFilename(lastIteration), emlib.MDL_RESOLUTION_FREQ, emlib.MDL_RESOLUTION_FRC ) From 5783f0cc21065c346b223c5bd706b7c316ef6447 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 30 Jan 2023 13:22:02 +0100 Subject: [PATCH 011/104] Refined output generation --- .../protocol_reconstruct_swiftres.py | 63 +++++++++++++++---- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index e0a1a014f..abe3c82ab 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -277,6 +277,38 @@ def filterVolumeStep(self, iteration: int): self.runJob('xmipp_transform_filter', args, numberOfMpi=1) def createOutputStep(self): + lastIteration = 0 # TODO + + # Keep only the image and id from the input particle set + args = [] + args += ['-i', self._getInputParticleMdFilename()] + args += ['-o', self._getOutputParticlesMdFilename()] + args += ['--operate', 'keep_column', 'itemId image'] + self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) + + # Add the rest from the last alignment + args = [] + args += ['-i', self._getOutputParticlesMdFilename()] + args += ['-o', self._getOutputParticlesMdFilename()] + args += ['--set', 'join', self._getAlignmentMdFilename(lastIteration), 'itemId'] + self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) + + # Link last iteration + for i in range(1, 2): + createLink( + self._getHalfVolumeFilename(lastIteration, i), + self._getOutputHalfVolumeFilename(i) + ) + createLink( + self._getFilteredVolumeFilename(lastIteration), + self._getOutputVolumeFilename() + ) + createLink( + self._getFscFilename(lastIteration), + self._getOutputFscFilename() + ) + + # Create output objects self._createOutputParticleSet() self._createOutputVolume() self._createOutputFsc() @@ -343,6 +375,17 @@ def _getFscFilename(self, iteration: int): def _getFilteredVolumeFilename(self, iteration: int): return self._getIterationPath(iteration, 'volume_filtered.mrc') + def _getOutputParticlesMdFilename(self): + return self._getExtraPath('output_particles.xmd') + + def _getOutputVolumeFilename(self): + return self._getExtraPath('output_volume.mrc') + + def _getOutputHalfVolumeFilename(self, half: int): + return self._getExtraPath('output_volume_half%01d.mrc' % half) + + def _getOutputFscFilename(self): + return self._getExtraPath('output_fsc.xmd') def _computeResolution(self, mdFsc, Ts, threshold): resolution = 2 * Ts @@ -360,12 +403,11 @@ def _createOutputVolume(self): volume=Volume() # Fill - lastIteration = 0 # TODO - volume.setFileName(self._getAverageVolumeFilename(lastIteration)) + volume.setFileName(self._getOutputVolumeFilename()) volume.setSamplingRate(self._getSamplingRate()) volume.setHalfMaps([ - self._getHalfVolumeFilename(lastIteration, 1), - self._getHalfVolumeFilename(lastIteration, 2) + self._getOutputHalfVolumeFilename(1), + self._getOutputHalfVolumeFilename(2), ]) # Define the output @@ -377,11 +419,11 @@ def _createOutputVolume(self): def _createOutputParticleSet(self): particleSet = self._createSetOfParticles() - # TODO replace wiener corrected images - # Fill - lastIteration = 0 # TODO - readSetOfParticles(self._getAlignmentMdFilename(lastIteration), particleSet) + readSetOfParticles( + self._getOutputParticlesMdFilename(), + particleSet, + ) particleSet.setSamplingRate(self._getSamplingRate()) # Define the output @@ -394,9 +436,8 @@ def _createOutputFsc(self): fsc = FSC() # Load from metadata - lastIteration = 0 # TODO fsc.loadFromMd( - self._getFscFilename(lastIteration), + self._getOutputFscFilename(), emlib.MDL_RESOLUTION_FREQ, emlib.MDL_RESOLUTION_FRC ) @@ -406,4 +447,4 @@ def _createOutputFsc(self): self._defineSourceRelation(self.inputParticles.get(), fsc) return fsc - \ No newline at end of file + \ No newline at end of file From 85178ae7e2e63dacd98c4b40fd419e6c8de0e3f7 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 31 Jan 2023 11:04:53 +0100 Subject: [PATCH 012/104] Added iterations --- .../protocol_reconstruct_swiftres.py | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index abe3c82ab..584c09be4 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -54,7 +54,6 @@ def _defineParams(self, form: Form): help="Add a list of GPU devices that can be used") form.addSection(label='Input') - form.addParam('inputParticles', PointerParam, label="Particles", important=True, pointerClass='SetOfParticles') form.addParam('inputVolume', PointerParam, label="Initial volumes", important=True, @@ -64,11 +63,18 @@ def _defineParams(self, form: Form): help='If no symmetry is present, give c1') form.addSection(label='Refinement') - form.addParam('resolutionLimit', FloatParam, label="Resolution limit (A)", default=10.0) + form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) + form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0) + form.addParam('nextResolutionCriterion', FloatParam, label="FSC criterion", default=0.5, expertLevel=LEVEL_ADVANCED, + help='The resolution of the reconstruction is defined as the inverse of the frequency at which '\ + 'the FSC drops below this value. Typical values are 0.143 and 0.5' ) form.addParam('angularSampling', FloatParam, label="Angular sampling (º)", default=5.0) form.addParam('shiftCount', IntParam, label="Shifts", default=9) form.addParam('maxShift', FloatParam, label="Maximum shift (%)", default=10.0) + form.addSection(label='Compute') + form.addParam('databaseItemCount', IntParam, label='Database item count', default=int(2e6)) + form.addParallelSection(threads=1, mpi=8) #--------------------------- INFO functions -------------------------------------------- @@ -80,7 +86,7 @@ def _insertAllSteps(self): correctCtfStepId = self._insertFunctionStep('correctCtfStep', prerequisites=[convertInputStepId]) lastIds = [correctCtfStepId] - for i in range(1): # TODO + for i in range(int(self.numberOfIterations)): lastIds = self._insertIterationSteps(i, prerequisites=lastIds) self._insertFunctionStep('createOutputStep', prerequisites=lastIds) @@ -107,8 +113,11 @@ def _insertReconstructSteps(self, iteration: int, prerequisites): return [computeFscStepId, averageVolumeStepId] def _insertPostProcessSteps(self, iteration: int, prerequisites): + """ filterVolumeStepId = self._insertFunctionStep('filterVolumeStep', iteration, prerequisites=prerequisites) return [filterVolumeStepId] + """ + return prerequisites #--------------------------- STEPS functions -------------------------------------------- def convertInputStep(self): @@ -144,15 +153,15 @@ def projectVolumeStep(self, iteration: int): self.runJob('xmipp_angular_project_library', args) def trainDatabaseStep(self, iteration: int): - expectedSize = int(2e6) # TODO determine form gallery - trainingSize = int(2e6) # TODO idem + expectedSize = int(self.databaseItemCount) + trainingSize = expectedSize args = [] args += ['-i', self._getGalleryMdFilename(iteration)] args += ['-o', self._getTrainingIndexFilename(iteration)] #args += ['--weights', self._getWeightsFilename(iteration)] args += ['--max_shift', self._getMaxShift()] - args += ['--max_frequency', self._getDigitalFrequencyLimit()] + args += ['--max_frequency', self._getIterationDigitalFrequencyLimit(iteration)] args += ['--method', 'fourier'] args += ['--size', expectedSize] args += ['--training', trainingSize] @@ -177,7 +186,7 @@ def alignStep(self, iteration: int): args += ['--max_shift', self._getMaxShift()] args += ['--rotations', nRotations] args += ['--shifts', nShift] - args += ['--max_frequency', self._getDigitalFrequencyLimit()] + args += ['--max_frequency', self._getIterationDigitalFrequencyLimit(iteration)] args += ['--method', 'fourier'] args += ['--dropna'] args += ['--batch', batchSize] @@ -277,7 +286,7 @@ def filterVolumeStep(self, iteration: int): self.runJob('xmipp_transform_filter', args, numberOfMpi=1) def createOutputStep(self): - lastIteration = 0 # TODO + lastIteration = int(self.numberOfIterations) - 1 # Keep only the image and id from the input particle set args = [] @@ -300,7 +309,7 @@ def createOutputStep(self): self._getOutputHalfVolumeFilename(i) ) createLink( - self._getFilteredVolumeFilename(lastIteration), + self._getAverageVolumeFilename(lastIteration), # TODO replace with post-processed volume self._getOutputVolumeFilename() ) createLink( @@ -315,14 +324,23 @@ def createOutputStep(self): #--------------------------- UTILS functions -------------------------------------------- - def _getDigitalFrequencyLimit(self): - return self.inputParticles.get().getSamplingRate() / float(self.resolutionLimit) - - def _getMaxShift(self): + def _getMaxShift(self) -> float: return float(self.maxShift) / 100.0 - def _getSamplingRate(self): - return self.inputParticles.get().getSamplingRate() + def _getSamplingRate(self) -> float: + return float(self.inputParticles.get().getSamplingRate()) + + def _getIterationResolutionLimit(self, iteration: int) -> float: + if iteration > 0: + mdFsc = emlib.MetaData(self._getFscFilename(iteration-1)) + sampling = self._getSamplingRate() + threshold = float(self.nextResolutionCriterion) + return self._computeResolution(mdFsc, sampling, threshold) + else: + return float(self.initialResolution) + + def _getIterationDigitalFrequencyLimit(self, iteration: int) -> float: + return self._getSamplingRate() / self._getIterationResolutionLimit(iteration) def _getIterationPath(self, iteration: int, *paths): return self._getExtraPath('iteration_%04d' % iteration, *paths) @@ -340,10 +358,10 @@ def _getInputVolumeFilename(self): return self.inputVolume.get().getFileName() def _getIterationInputVolumeFilename(self, iteration: int): - if iteration == 0: - return self._getInputVolumeFilename() + if iteration > 0: + return self._getAverageVolumeFilename(iteration-1) # TODO replace with post processed volume else: - return self._getAverageVolumeFilename(iteration-1) + return self._getInputVolumeFilename() def _getGalleryMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'gallery.doc') From e81cb9914c909d5fb35492ff8eaf453eb0b36b67 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 31 Jan 2023 12:10:20 +0100 Subject: [PATCH 013/104] Added scratch file --- .../protocols/protocol_reconstruct_swiftres.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 584c09be4..55ef5753d 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -73,7 +73,10 @@ def _defineParams(self, form: Form): form.addParam('maxShift', FloatParam, label="Maximum shift (%)", default=10.0) form.addSection(label='Compute') - form.addParam('databaseItemCount', IntParam, label='Database item count', default=int(2e6)) + form.addParam('databaseRecipe', StringParam, label='Database recipe', + default='OPQ48_192,IVF32768,PQ48' ) + form.addParam('databaseTrainingSetSize', IntParam, label='Database training set size', + default=int(2e6) ) form.addParallelSection(threads=1, mpi=8) @@ -153,18 +156,19 @@ def projectVolumeStep(self, iteration: int): self.runJob('xmipp_angular_project_library', args) def trainDatabaseStep(self, iteration: int): - expectedSize = int(self.databaseItemCount) - trainingSize = expectedSize + trainingSize = int(self.databaseTrainingSetSize) + recipe = self.databaseRecipe args = [] args += ['-i', self._getGalleryMdFilename(iteration)] args += ['-o', self._getTrainingIndexFilename(iteration)] + args += ['--recipe', recipe] #args += ['--weights', self._getWeightsFilename(iteration)] args += ['--max_shift', self._getMaxShift()] args += ['--max_frequency', self._getIterationDigitalFrequencyLimit(iteration)] args += ['--method', 'fourier'] - args += ['--size', expectedSize] args += ['--training', trainingSize] + args += ['--scratch', self._getTrainingScratchFilename()] if self.useGpu: args += ['--gpu', 0] # TODO select @@ -303,7 +307,7 @@ def createOutputStep(self): self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) # Link last iteration - for i in range(1, 2): + for i in range(1, 3): createLink( self._getHalfVolumeFilename(lastIteration, i), self._getOutputHalfVolumeFilename(i) @@ -405,6 +409,9 @@ def _getOutputHalfVolumeFilename(self, half: int): def _getOutputFscFilename(self): return self._getExtraPath('output_fsc.xmd') + def _getTrainingScratchFilename(self): + return self._getTmpPath('scratch.bin') + def _computeResolution(self, mdFsc, Ts, threshold): resolution = 2 * Ts From 65ef2a7518b3b8ffbd18504d5426d3e7e9971340 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 1 Feb 2023 09:18:03 +0100 Subject: [PATCH 014/104] Added reprojection comparison --- .../protocol_reconstruct_swiftres.py | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 55ef5753d..d017cb019 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -105,7 +105,8 @@ def _insertAlignmentSteps(self, iteration: int, prerequisites): projectVolumeStepId = self._insertFunctionStep('projectVolumeStep', iteration, prerequisites=prerequisites) trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=[projectVolumeStepId]) alignStepId = self._insertFunctionStep('alignStep', iteration, prerequisites=[trainDatabaseStepId]) - return [alignStepId] + compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, prerequisites=[alignStepId]) + return [compareReprojectionStepId] def _insertReconstructSteps(self, iteration: int, prerequisites): splitStepId = self._insertFunctionStep('splitStep', iteration, prerequisites=prerequisites) @@ -201,6 +202,19 @@ def alignStep(self, iteration: int): env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) + def compareReprojectionStep(self, iteration: int): + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--ref', self._getIterationInputVolumeFilename(iteration)] + args += ['--ignoreCTF'] # As we're using wiener corrected images + args += ['--doNotWriteStack'] # Do not undo shifts + self.runJob('xmipp_angular_continuous_assign2', args, numberOfMpi=self.numberOfMpi.get()) + + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--operate', 'rename_column', 'weightContinuous2 weight'] + self._runMdUtils(args) + def splitStep(self, iteration: int): args = [] args += ['-i', self._getAlignmentMdFilename(iteration)] @@ -294,17 +308,16 @@ def createOutputStep(self): # Keep only the image and id from the input particle set args = [] - args += ['-i', self._getInputParticleMdFilename()] + args += ['-i', self._getAlignmentMdFilename(lastIteration)] args += ['-o', self._getOutputParticlesMdFilename()] - args += ['--operate', 'keep_column', 'itemId image'] - self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) + args += ['--operate', 'rename_column', 'image image1'] + self._runMdUtils(args) # Add the rest from the last alignment args = [] args += ['-i', self._getOutputParticlesMdFilename()] - args += ['-o', self._getOutputParticlesMdFilename()] - args += ['--set', 'join', self._getAlignmentMdFilename(lastIteration), 'itemId'] - self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) + args += ['--set', 'join', self._getInputParticleMdFilename(), 'itemId'] + self._runMdUtils(args) # Link last iteration for i in range(1, 3): @@ -444,10 +457,22 @@ def _createOutputVolume(self): def _createOutputParticleSet(self): particleSet = self._createSetOfParticles() + """ + EXTRA_LABELS = [ + emlib.MDL_COST, + emlib.MDL_WEIGHT, + emlib.MDL_CORRELATION_IDX, + emlib.MDL_CORRELATION_MASK, + emlib.MDL_CORRELATION_WEIGHT, + emlib.MDL_IMED + ] + """ + # Fill readSetOfParticles( self._getOutputParticlesMdFilename(), - particleSet, + particleSet + #extraLabels=EXTRA_LABELS ) particleSet.setSamplingRate(self._getSamplingRate()) @@ -472,4 +497,8 @@ def _createOutputFsc(self): self._defineSourceRelation(self.inputParticles.get(), fsc) return fsc + + def _runMdUtils(self, args): + self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) + \ No newline at end of file From f7aa385ce19fd51bfab028f883f9275d77e16f3f Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 1 Feb 2023 12:23:19 +0100 Subject: [PATCH 015/104] Added compute weights step --- .../protocol_reconstruct_swiftres.py | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index d017cb019..46a941f93 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -105,11 +105,13 @@ def _insertAlignmentSteps(self, iteration: int, prerequisites): projectVolumeStepId = self._insertFunctionStep('projectVolumeStep', iteration, prerequisites=prerequisites) trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=[projectVolumeStepId]) alignStepId = self._insertFunctionStep('alignStep', iteration, prerequisites=[trainDatabaseStepId]) - compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, prerequisites=[alignStepId]) + compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, prerequisites=[alignStepId]) + compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, prerequisites=[compareAnglesStepId]) return [compareReprojectionStepId] def _insertReconstructSteps(self, iteration: int, prerequisites): - splitStepId = self._insertFunctionStep('splitStep', iteration, prerequisites=prerequisites) + computeWeightsStepId = self._insertFunctionStep('computeWeightsStep', iteration, prerequisites=prerequisites) + splitStepId = self._insertFunctionStep('splitStep', iteration, prerequisites=[computeWeightsStepId]) reconstructStepId1 = self._insertFunctionStep('reconstructStep', iteration, 1, prerequisites=[splitStepId]) reconstructStepId2 = self._insertFunctionStep('reconstructStep', iteration, 2, prerequisites=[splitStepId]) computeFscStepId = self._insertFunctionStep('computeFscStep', iteration, prerequisites=[reconstructStepId1, reconstructStepId2]) @@ -201,6 +203,19 @@ def alignStep(self, iteration: int): env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) + + def compareAnglesStep(self, iteration: int): + args = [] + args += ['--ang1', self._getAlignmentMdFilename(iteration)] + args += ['--ang2', self._getInputParticleMdFilename()] + args += ['--oroot', self._getAngleDiffOutputRoot(iteration)] + args += ['--sym', self.symmetryGroup] + self.runJob('xmipp_angular_distance', args, numberOfMpi=1) + + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--set', 'join', self._getAngleDiffMdFilename(iteration)] + self._runMdUtils(args) def compareReprojectionStep(self, iteration: int): args = [] @@ -210,11 +225,24 @@ def compareReprojectionStep(self, iteration: int): args += ['--doNotWriteStack'] # Do not undo shifts self.runJob('xmipp_angular_continuous_assign2', args, numberOfMpi=self.numberOfMpi.get()) + def computeWeightsStep(self, iteration: int): + """ + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--fill', 'weight', 'constant', '0.0'] + self._runMdUtils(args) + + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--operate', 'modify_values', 'weight=corrIdx*corrWeight*corrMask'] + self._runMdUtils(args) + """ + args = [] args += ['-i', self._getAlignmentMdFilename(iteration)] args += ['--operate', 'rename_column', 'weightContinuous2 weight'] self._runMdUtils(args) - + def splitStep(self, iteration: int): args = [] args += ['-i', self._getAlignmentMdFilename(iteration)] @@ -395,6 +423,12 @@ def _getTrainingIndexFilename(self, iteration: int): def _getAlignmentMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'aligned.xmd') + def _getAngleDiffOutputRoot(self, iteration: int, suffix=''): + return self._getIterationPath(iteration, 'angles'+suffix) + + def _getAngleDiffMdFilename(self, iteration: int): + return self._getAngleDiffOutputRoot(iteration, '.xmd') + def _getAlignmentHalfMdFilename(self, iteration: int, half: int): return self._getIterationPath(iteration, 'aligned%06d.xmd' % half) From 739de7e4d341c36bffeee71c042e0a62f68a3479 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 1 Feb 2023 12:55:07 +0100 Subject: [PATCH 016/104] Bugfix --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 46a941f93..b8af2c5c0 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -205,9 +205,15 @@ def alignStep(self, iteration: int): self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) def compareAnglesStep(self, iteration: int): + args = [] + args += ['-i', self._getInputParticleMdFilename()] + args += ['-o', self._getInputIntersectionMdFilename(iteration)] + args += ['--set', 'intersection', self._getAlignmentMdFilename(iteration), 'itemId'] + self._runMdUtils(args) + args = [] args += ['--ang1', self._getAlignmentMdFilename(iteration)] - args += ['--ang2', self._getInputParticleMdFilename()] + args += ['--ang2', self._getInputIntersectionMdFilename(iteration)] args += ['--oroot', self._getAngleDiffOutputRoot(iteration)] args += ['--sym', self.symmetryGroup] self.runJob('xmipp_angular_distance', args, numberOfMpi=1) @@ -423,6 +429,9 @@ def _getTrainingIndexFilename(self, iteration: int): def _getAlignmentMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'aligned.xmd') + def _getInputIntersectionMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'input_intersection.xmd') + def _getAngleDiffOutputRoot(self, iteration: int, suffix=''): return self._getIterationPath(iteration, 'angles'+suffix) From e9024450948b02b08e82c39a7edca9053fc386bd Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 1 Feb 2023 15:02:15 +0100 Subject: [PATCH 017/104] Started working on multiclass --- .../protocol_reconstruct_swiftres.py | 259 +++++++++++------- 1 file changed, 165 insertions(+), 94 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index b8af2c5c0..4b8142b12 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -95,27 +95,46 @@ def _insertAllSteps(self): self._insertFunctionStep('createOutputStep', prerequisites=lastIds) def _insertIterationSteps(self, iteration: int, prerequisites): + setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, prerequisites=prerequisites) - alignIds = self._insertAlignmentSteps(iteration, prerequisites=[setupIterationStepId]) - reconstructIds = self._insertReconstructSteps(iteration, prerequisites=alignIds) - postProcessIds = self._insertPostProcessSteps(iteration, prerequisites=reconstructIds) - return postProcessIds + projectIds = self._insertProjectSteps(iteration, prerequisites=[setupIterationStepId]) + alignIds = self._insertAlignmentSteps(iteration, prerequisites=projectIds) + + ids = [] + for cls in range(self._getClassCount()): + reconstructIds = self._insertReconstructSteps(iteration, cls, prerequisites=alignIds) + postProcessIds = self._insertPostProcessSteps(iteration, cls, prerequisites=reconstructIds) + ids += postProcessIds + + return ids + + def _insertProjectSteps(self, iteration: int, prerequisites): + # Project all volumes + projectStepIds = [] + for cls in range(self._getClassCount()): + projectStepIds.append(self._insertFunctionStep('projectVolumeStep', iteration, cls, prerequisites=prerequisites)) + + # Merge galleries + mergeGalleriesStepId = self._insertFunctionStep('mergeGalleriesStep', iteration, prerequisites=projectStepIds) + + return [mergeGalleriesStepId] def _insertAlignmentSteps(self, iteration: int, prerequisites): - projectVolumeStepId = self._insertFunctionStep('projectVolumeStep', iteration, prerequisites=prerequisites) - trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=[projectVolumeStepId]) + trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=prerequisites) alignStepId = self._insertFunctionStep('alignStep', iteration, prerequisites=[trainDatabaseStepId]) - compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, prerequisites=[alignStepId]) - compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, prerequisites=[compareAnglesStepId]) - return [compareReprojectionStepId] + + return [alignStepId] - def _insertReconstructSteps(self, iteration: int, prerequisites): - computeWeightsStepId = self._insertFunctionStep('computeWeightsStep', iteration, prerequisites=prerequisites) - splitStepId = self._insertFunctionStep('splitStep', iteration, prerequisites=[computeWeightsStepId]) - reconstructStepId1 = self._insertFunctionStep('reconstructStep', iteration, 1, prerequisites=[splitStepId]) - reconstructStepId2 = self._insertFunctionStep('reconstructStep', iteration, 2, prerequisites=[splitStepId]) - computeFscStepId = self._insertFunctionStep('computeFscStep', iteration, prerequisites=[reconstructStepId1, reconstructStepId2]) - averageVolumeStepId = self._insertFunctionStep('averageVolumeStep', iteration, prerequisites=[reconstructStepId1, reconstructStepId2]) + def _insertReconstructSteps(self, iteration: int, prerequisites, cls: int): + selectAlignmentStepId = self._insertFunctionStep('selectAlignmentStep', iteration, cls, prerequisites=prerequisites) + compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, cls, prerequisites=[selectAlignmentStepId]) + compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[compareAnglesStepId]) + computeWeightsStepId = self._insertFunctionStep('computeWeightsStep', iteration, cls, prerequisites=[compareReprojectionStepId]) + splitStepId = self._insertFunctionStep('splitStep', iteration, cls, prerequisites=[computeWeightsStepId]) + reconstructStepId1 = self._insertFunctionStep('reconstructStep', iteration, cls, 1, prerequisites=[splitStepId]) + reconstructStepId2 = self._insertFunctionStep('reconstructStep', iteration, cls, 2, prerequisites=[splitStepId]) + computeFscStepId = self._insertFunctionStep('computeFscStep', iteration, cls, prerequisites=[reconstructStepId1, reconstructStepId2]) + averageVolumeStepId = self._insertFunctionStep('averageVolumeStep', iteration, cls, prerequisites=[reconstructStepId1, reconstructStepId2]) return [computeFscStepId, averageVolumeStepId] def _insertPostProcessSteps(self, iteration: int, prerequisites): @@ -148,16 +167,44 @@ def correctCtfStep(self): def setupIterationStep(self, iteration: int): makePath(self._getIterationPath(iteration)) + + for cls in range(self._getClassCount()): + makePath(self._getClassPath(iteration, cls)) - def projectVolumeStep(self, iteration: int): + def projectVolumeStep(self, iteration: int, cls: int): args = [] - args += ['-i', self._getIterationInputVolumeFilename(iteration)] - args += ['-o', self._getGalleryStackFilename(iteration)] + args += ['-i', self._getIterationInputVolumeFilename(iteration, cls)] + args += ['-o', self._getClassGalleryStackFilename(iteration, cls)] args += ['--sampling_rate', self.angularSampling] args += ['--sym', self.symmetryGroup] self.runJob('xmipp_angular_project_library', args) + args = [] + args += ['-i', self._getClassGalleryMdFilename(iteration, cls)] + args += ['--fill', 'classId', 'constant', cls] + self._runMdUtils(args) + + def mergeGalleriesStep(self, iteration): + # Copy the first gallery + copyFile( + self._getClassGalleryMdFilename(iteration, 0), + self._getGalleryMdFilename(iteration) + ) + + # Merge subsequent galleries + for cls in range(1, self._getClassCount()): + args = [] + args += ['-i', self._getGalleryMdFilename(iteration)] + args += ['--set', 'union', self._getClassGalleryMdFilename(iteration, cls)] + self._runMdUtils(args) + + # Reindex + args = [] + args += ['-i', self._getGalleryMdFilename(iteration)] + args += ['--fill', 'lineal', 1, 1] + self._runMdUtils(args) + def trainDatabaseStep(self, iteration: int): trainingSize = int(self.databaseTrainingSetSize) recipe = self.databaseRecipe @@ -204,34 +251,41 @@ def alignStep(self, iteration: int): env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) - def compareAnglesStep(self, iteration: int): + def selectAlignmentStep(self, iteration: int, cls: int): + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['-o', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['--query', 'select', 'classId==%d' % cls] + self._runMdUtils(args) + + def compareAnglesStep(self, iteration: int, cls): args = [] args += ['-i', self._getInputParticleMdFilename()] - args += ['-o', self._getInputIntersectionMdFilename(iteration)] - args += ['--set', 'intersection', self._getAlignmentMdFilename(iteration), 'itemId'] + args += ['-o', self._getInputIntersectionMdFilename(iteration, cls)] + args += ['--set', 'intersection', self._getClassAlignmentMdFilename(iteration, cls), 'itemId'] self._runMdUtils(args) args = [] - args += ['--ang1', self._getAlignmentMdFilename(iteration)] - args += ['--ang2', self._getInputIntersectionMdFilename(iteration)] - args += ['--oroot', self._getAngleDiffOutputRoot(iteration)] + args += ['--ang1', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['--ang2', self._getInputIntersectionMdFilename(iteration, cls)] + args += ['--oroot', self._getAngleDiffOutputRoot(iteration, cls)] args += ['--sym', self.symmetryGroup] self.runJob('xmipp_angular_distance', args, numberOfMpi=1) args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--set', 'join', self._getAngleDiffMdFilename(iteration)] + args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['--set', 'join', self._getAngleDiffMdFilename(iteration, cls)] self._runMdUtils(args) - def compareReprojectionStep(self, iteration: int): + def compareReprojectionStep(self, iteration: int, cls: int): args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--ref', self._getIterationInputVolumeFilename(iteration)] + args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['--ref', self._getIterationInputVolumeFilename(iteration, cls)] args += ['--ignoreCTF'] # As we're using wiener corrected images args += ['--doNotWriteStack'] # Do not undo shifts self.runJob('xmipp_angular_continuous_assign2', args, numberOfMpi=self.numberOfMpi.get()) - def computeWeightsStep(self, iteration: int): + def computeWeightsStep(self, iteration: int, cls: int): """ args = [] args += ['-i', self._getAlignmentMdFilename(iteration)] @@ -245,21 +299,21 @@ def computeWeightsStep(self, iteration: int): """ args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] args += ['--operate', 'rename_column', 'weightContinuous2 weight'] self._runMdUtils(args) - def splitStep(self, iteration: int): + def splitStep(self, iteration: int, cls: int): args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] args += ['-n', 2] self.runJob('xmipp_metadata_split', args, numberOfMpi=1) - def reconstructStep(self, iteration: int, half: int): + def reconstructStep(self, iteration: int, cls: int, half: int): args = [] - args += ['-i', self._getAlignmentHalfMdFilename(iteration, half)] - args += ['-o', self._getHalfVolumeFilename(iteration, half)] + args += ['-i', self._getClassAlignmentHalfMdFilename(iteration, cls, half)] + args += ['-o', self._getHalfVolumeFilename(iteration, cls, half)] args += ['--sym', self.symmetryGroup.get()] args += ['--weight'] @@ -304,20 +358,20 @@ def reconstructStep(self, iteration: int, half: int): self.runJob(reconstructProgram, args, numberOfMpi=numberOfMpi) - def computeFscStep(self, iteration: int): + def computeFscStep(self, iteration: int, cls: int): args = [] - args += ['--ref', self._getHalfVolumeFilename(iteration, 1)] - args += ['-i', self._getHalfVolumeFilename(iteration, 2)] - args += ['-o', self._getFscFilename(iteration)] + args += ['--ref', self._getHalfVolumeFilename(iteration, cls, 1)] + args += ['-i', self._getHalfVolumeFilename(iteration, cls, 2)] + args += ['-o', self._getFscFilename(iteration, cls)] args += ['--sampling_rate', self._getSamplingRate()] self.runJob('xmipp_resolution_fsc', args, numberOfMpi=1) - def averageVolumeStep(self, iteration: int): + def averageVolumeStep(self, iteration: int, cls: int): args = [] - args += ['-i', self._getHalfVolumeFilename(iteration, 1)] - args += ['--plus', self._getHalfVolumeFilename(iteration, 2)] - args += ['-o', self._getAverageVolumeFilename(iteration)] + args += ['-i', self._getHalfVolumeFilename(iteration, cls, 1)] + args += ['--plus', self._getHalfVolumeFilename(iteration, cls, 2)] + args += ['-o', self._getAverageVolumeFilename(iteration, cls)] self.runJob('xmipp_image_operate', args, numberOfMpi=1) args = [] @@ -325,13 +379,13 @@ def averageVolumeStep(self, iteration: int): args += ['--mult', '0.5'] self.runJob('xmipp_image_operate', args, numberOfMpi=1) - def filterVolumeStep(self, iteration: int): - mdFsc = emlib.MetaData(self._getFscFilename(iteration)) + def filterVolumeStep(self, iteration: int, cls: int): + mdFsc = emlib.MetaData(self._getFscFilename(iteration, cls)) resolution = self._computeResolution(mdFsc, self._getSamplingRate(), 0.5) args = [] - args += ['-i', self._getAverageVolumeFilename(iteration)] - args += ['-o', self._getFilteredVolumeFilename(iteration)] + args += ['-i', self._getAverageVolumeFilename(iteration, cls)] + args += ['-o', self._getFilteredVolumeFilename(iteration, cls)] args += ['--fourier', 'low_pass', resolution] args += ['--sampling', self._getSamplingRate()] @@ -354,19 +408,21 @@ def createOutputStep(self): self._runMdUtils(args) # Link last iteration - for i in range(1, 3): + for cls in range(self._getClassCount()): + for i in range(1, 3): + createLink( + self._getHalfVolumeFilename(lastIteration, cls, i), + self._getOutputHalfVolumeFilename(cls, i) + ) + createLink( - self._getHalfVolumeFilename(lastIteration, i), - self._getOutputHalfVolumeFilename(i) + self._getAverageVolumeFilename(cls, lastIteration), # TODO replace with post-processed volume + self._getOutputVolumeFilename(cls) + ) + createLink( + self._getFscFilename(cls, lastIteration), + self._getOutputFscFilename(cls) ) - createLink( - self._getAverageVolumeFilename(lastIteration), # TODO replace with post-processed volume - self._getOutputVolumeFilename() - ) - createLink( - self._getFscFilename(lastIteration), - self._getOutputFscFilename() - ) # Create output objects self._createOutputParticleSet() @@ -375,6 +431,9 @@ def createOutputStep(self): #--------------------------- UTILS functions -------------------------------------------- + def _getClassCount(self) -> int: + return 1 + def _getMaxShift(self) -> float: return float(self.maxShift) / 100.0 @@ -396,6 +455,9 @@ def _getIterationDigitalFrequencyLimit(self, iteration: int) -> float: def _getIterationPath(self, iteration: int, *paths): return self._getExtraPath('iteration_%04d' % iteration, *paths) + def _getClassPath(self, iteration: int, cls: int, *paths): + return self._getIterationPath(iteration, 'class_%06d' % cls, *paths) + def _getInputParticleMdFilename(self): return self._getExtraPath('input_particles.xmd') @@ -405,21 +467,24 @@ def _getWienerParticleMdFilename(self): def _getWienerParticleStackFilename(self): return self._getExtraPath('input_particles_wiener.mrcs') - def _getInputVolumeFilename(self): - return self.inputVolume.get().getFileName() + def _getInputVolumeFilename(self, cls: int): + return self.inputVolume.get().getFileName() # TODO consider class - def _getIterationInputVolumeFilename(self, iteration: int): + def _getIterationInputVolumeFilename(self, iteration: int, cls: int): if iteration > 0: - return self._getAverageVolumeFilename(iteration-1) # TODO replace with post processed volume + return self._getAverageVolumeFilename(iteration-1, cls) # TODO replace with post processed volume else: - return self._getInputVolumeFilename() + return self._getInputVolumeFilename(cls) + + def _getClassGalleryMdFilename(self, iteration: int, cls: int): + return self._getClassPath(iteration, cls, 'gallery.doc') + + def _getClassGalleryStackFilename(self, iteration: int, cls: int): + return self._getClassPath(iteration, cls, 'gallery.mrcs') def _getGalleryMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'gallery.doc') - def _getGalleryStackFilename(self, iteration: int): - return self._getIterationPath(iteration, 'gallery.mrcs') - def _getWeightsFilename(self, iteration: int): return self._getIterationPath(iteration, 'weights.mrc') @@ -429,41 +494,44 @@ def _getTrainingIndexFilename(self, iteration: int): def _getAlignmentMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'aligned.xmd') - def _getInputIntersectionMdFilename(self, iteration: int): - return self._getIterationPath(iteration, 'input_intersection.xmd') + def _getInputIntersectionMdFilename(self, iteration: int, cls: int): + return self._getClassPath(iteration, cls, 'input_intersection.xmd') - def _getAngleDiffOutputRoot(self, iteration: int, suffix=''): - return self._getIterationPath(iteration, 'angles'+suffix) + def _getAngleDiffOutputRoot(self, iteration: int, cls: int, suffix=''): + return self._getClassPath(iteration, cls, 'angles'+suffix) + + def _getAngleDiffMdFilename(self, iteration: int, cls: int): + return self._getAngleDiffOutputRoot(iteration, cls, '.xmd') - def _getAngleDiffMdFilename(self, iteration: int): - return self._getAngleDiffOutputRoot(iteration, '.xmd') + def _getClassAlignmentMdFilename(self, iteration: int, cls: int): + return self._getClassPath(iteration, cls, 'aligned.xmd') - def _getAlignmentHalfMdFilename(self, iteration: int, half: int): - return self._getIterationPath(iteration, 'aligned%06d.xmd' % half) + def _getClassAlignmentHalfMdFilename(self, iteration: int, cls: int, half: int): + return self._getClassPath(iteration, cls, 'aligned%06d.xmd' % half) - def _getHalfVolumeFilename(self, iteration: int, half: int): - return self._getIterationPath(iteration, 'volume_half%01d.mrc' % half) + def _getHalfVolumeFilename(self, iteration: int, cls: int, half: int): + return self._getClassPath(iteration, cls, 'volume_half%01d.mrc' % half) - def _getAverageVolumeFilename(self, iteration: int): - return self._getIterationPath(iteration, 'volume_avg.mrc') + def _getAverageVolumeFilename(self, iteration: int, cls: int): + return self._getClassPath(iteration, cls, 'volume_avg.mrc') - def _getFscFilename(self, iteration: int): - return self._getIterationPath(iteration, 'fsc.xmd') + def _getFscFilename(self, iteration: int, cls: int): + return self._getClassPath(iteration, cls, 'fsc.xmd') - def _getFilteredVolumeFilename(self, iteration: int): - return self._getIterationPath(iteration, 'volume_filtered.mrc') + def _getFilteredVolumeFilename(self, iteration: int, cls: int): + return self._getClassPath(iteration, cls, 'volume_filtered.mrc') def _getOutputParticlesMdFilename(self): return self._getExtraPath('output_particles.xmd') - def _getOutputVolumeFilename(self): - return self._getExtraPath('output_volume.mrc') + def _getOutputVolumeFilename(self, cls: int): + return self._getExtraPath('output_volume%06d.mrc' % cls) - def _getOutputHalfVolumeFilename(self, half: int): - return self._getExtraPath('output_volume_half%01d.mrc' % half) + def _getOutputHalfVolumeFilename(self, cls: int, half: int): + return self._getExtraPath('output_volume%06d_half%01d.mrc' % (cls, half)) - def _getOutputFscFilename(self): - return self._getExtraPath('output_fsc.xmd') + def _getOutputFscFilename(self, cls: int): + return self._getExtraPath('output_fsc%06d.xmd' % cls) def _getTrainingScratchFilename(self): return self._getTmpPath('scratch.bin') @@ -484,11 +552,12 @@ def _createOutputVolume(self): volume=Volume() # Fill - volume.setFileName(self._getOutputVolumeFilename()) + cls = 0 # TODO adapt for multiple volumes + volume.setFileName(self._getOutputVolumeFilename(cls)) volume.setSamplingRate(self._getSamplingRate()) volume.setHalfMaps([ - self._getOutputHalfVolumeFilename(1), - self._getOutputHalfVolumeFilename(2), + self._getOutputHalfVolumeFilename(cls, 1), + self._getOutputHalfVolumeFilename(cls, 2), ]) # Define the output @@ -498,6 +567,7 @@ def _createOutputVolume(self): return volume def _createOutputParticleSet(self): + # TODO adapt for classes 3D particleSet = self._createSetOfParticles() """ @@ -529,8 +599,9 @@ def _createOutputFsc(self): fsc = FSC() # Load from metadata + cls = 0 # TODO adapt for multiple FSCs fsc.loadFromMd( - self._getOutputFscFilename(), + self._getOutputFscFilename(cls), emlib.MDL_RESOLUTION_FREQ, emlib.MDL_RESOLUTION_FRC ) From 54b1af3a07002c9d843e993e2aa30d8a4e92917d Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 2 Feb 2023 09:03:59 +0100 Subject: [PATCH 018/104] Started working on 3D classification --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 4b8142b12..92bff08d9 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -125,7 +125,7 @@ def _insertAlignmentSteps(self, iteration: int, prerequisites): return [alignStepId] - def _insertReconstructSteps(self, iteration: int, prerequisites, cls: int): + def _insertReconstructSteps(self, iteration: int, cls: int, prerequisites): selectAlignmentStepId = self._insertFunctionStep('selectAlignmentStep', iteration, cls, prerequisites=prerequisites) compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, cls, prerequisites=[selectAlignmentStepId]) compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[compareAnglesStepId]) @@ -137,7 +137,7 @@ def _insertReconstructSteps(self, iteration: int, prerequisites, cls: int): averageVolumeStepId = self._insertFunctionStep('averageVolumeStep', iteration, cls, prerequisites=[reconstructStepId1, reconstructStepId2]) return [computeFscStepId, averageVolumeStepId] - def _insertPostProcessSteps(self, iteration: int, prerequisites): + def _insertPostProcessSteps(self, iteration: int, cls: int, prerequisites): """ filterVolumeStepId = self._insertFunctionStep('filterVolumeStep', iteration, prerequisites=prerequisites) return [filterVolumeStepId] @@ -182,7 +182,7 @@ def projectVolumeStep(self, iteration: int, cls: int): args = [] args += ['-i', self._getClassGalleryMdFilename(iteration, cls)] - args += ['--fill', 'classId', 'constant', cls] + args += ['--fill', 'ref3d', 'constant', cls] self._runMdUtils(args) def mergeGalleriesStep(self, iteration): @@ -202,7 +202,7 @@ def mergeGalleriesStep(self, iteration): # Reindex args = [] args += ['-i', self._getGalleryMdFilename(iteration)] - args += ['--fill', 'lineal', 1, 1] + args += ['--fill', 'ref', 'lineal', 0, 1] self._runMdUtils(args) def trainDatabaseStep(self, iteration: int): @@ -255,7 +255,7 @@ def selectAlignmentStep(self, iteration: int, cls: int): args = [] args += ['-i', self._getAlignmentMdFilename(iteration)] args += ['-o', self._getClassAlignmentMdFilename(iteration, cls)] - args += ['--query', 'select', 'classId==%d' % cls] + args += ['--query', 'select', 'ref3d==%d' % cls] self._runMdUtils(args) def compareAnglesStep(self, iteration: int, cls): @@ -375,7 +375,7 @@ def averageVolumeStep(self, iteration: int, cls: int): self.runJob('xmipp_image_operate', args, numberOfMpi=1) args = [] - args += ['-i', self._getAverageVolumeFilename(iteration)] + args += ['-i', self._getAverageVolumeFilename(iteration, cls)] args += ['--mult', '0.5'] self.runJob('xmipp_image_operate', args, numberOfMpi=1) @@ -432,7 +432,7 @@ def createOutputStep(self): #--------------------------- UTILS functions -------------------------------------------- def _getClassCount(self) -> int: - return 1 + return 1 # TODO replace def _getMaxShift(self) -> float: return float(self.maxShift) / 100.0 From 654a9ed3d335cd39d31a7731e40805435608f079 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 2 Feb 2023 12:09:18 +0100 Subject: [PATCH 019/104] Added SetOfClasses to the output --- .../protocol_reconstruct_swiftres.py | 134 ++++++++++-------- 1 file changed, 76 insertions(+), 58 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 92bff08d9..2e98086ff 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -21,12 +21,13 @@ # ***************************************************************************/ from pwem.protocols import ProtRefine3D -from pwem.objects import Volume, FSC +from pwem.objects import Volume, FSC, SetOfVolumes, Class3D from pwem import emlib from pyworkflow.protocol.params import (Form, PointerParam, FloatParam, IntParam, StringParam, BooleanParam, + MultiPointerParam, LEVEL_ADVANCED, USE_GPU, GPU_LIST ) from pyworkflow.utils.path import (cleanPath, makePath, copyFile, moveFile, createLink, cleanPattern) @@ -56,7 +57,7 @@ def _defineParams(self, form: Form): form.addSection(label='Input') form.addParam('inputParticles', PointerParam, label="Particles", important=True, pointerClass='SetOfParticles') - form.addParam('inputVolume', PointerParam, label="Initial volumes", important=True, + form.addParam('inputVolumes', MultiPointerParam, label="Initial volumes", important=True, pointerClass='Volume') form.addParam('symmetryGroup', StringParam, default="c1", label='Symmetry group', @@ -182,7 +183,7 @@ def projectVolumeStep(self, iteration: int, cls: int): args = [] args += ['-i', self._getClassGalleryMdFilename(iteration, cls)] - args += ['--fill', 'ref3d', 'constant', cls] + args += ['--fill', 'ref3d', 'constant', cls+1] self._runMdUtils(args) def mergeGalleriesStep(self, iteration): @@ -202,7 +203,7 @@ def mergeGalleriesStep(self, iteration): # Reindex args = [] args += ['-i', self._getGalleryMdFilename(iteration)] - args += ['--fill', 'ref', 'lineal', 0, 1] + args += ['--fill', 'ref', 'lineal', 1, 1] self._runMdUtils(args) def trainDatabaseStep(self, iteration: int): @@ -255,7 +256,7 @@ def selectAlignmentStep(self, iteration: int, cls: int): args = [] args += ['-i', self._getAlignmentMdFilename(iteration)] args += ['-o', self._getClassAlignmentMdFilename(iteration, cls)] - args += ['--query', 'select', 'ref3d==%d' % cls] + args += ['--query', 'select', 'ref3d==%d' % (cls+1)] self._runMdUtils(args) def compareAnglesStep(self, iteration: int, cls): @@ -425,14 +426,14 @@ def createOutputStep(self): ) # Create output objects - self._createOutputParticleSet() - self._createOutputVolume() - self._createOutputFsc() + volumes = self._createOutputVolumes() + self._createOutputClasses3D(volumes) + self._createOutputFscs() #--------------------------- UTILS functions -------------------------------------------- def _getClassCount(self) -> int: - return 1 # TODO replace + return len(self.inputVolumes) def _getMaxShift(self) -> float: return float(self.maxShift) / 100.0 @@ -468,7 +469,7 @@ def _getWienerParticleStackFilename(self): return self._getExtraPath('input_particles_wiener.mrcs') def _getInputVolumeFilename(self, cls: int): - return self.inputVolume.get().getFileName() # TODO consider class + return self.inputVolumes[cls].get().getFileName() def _getIterationInputVolumeFilename(self, iteration: int, cls: int): if iteration > 0: @@ -548,69 +549,86 @@ def _computeResolution(self, mdFsc, Ts, threshold): return resolution - def _createOutputVolume(self): - volume=Volume() - - # Fill - cls = 0 # TODO adapt for multiple volumes - volume.setFileName(self._getOutputVolumeFilename(cls)) - volume.setSamplingRate(self._getSamplingRate()) - volume.setHalfMaps([ - self._getOutputHalfVolumeFilename(cls, 1), - self._getOutputHalfVolumeFilename(cls, 2), - ]) - - # Define the output - self._defineOutputs(outputVolume=volume) - self._defineSourceRelation(self.inputParticles.get(), volume) - - return volume - - def _createOutputParticleSet(self): - # TODO adapt for classes 3D - particleSet = self._createSetOfParticles() - - """ + def _createOutputClasses3D(self, volumes: SetOfVolumes): + particles = self._createSetOfParticles() + EXTRA_LABELS = [ - emlib.MDL_COST, - emlib.MDL_WEIGHT, - emlib.MDL_CORRELATION_IDX, - emlib.MDL_CORRELATION_MASK, - emlib.MDL_CORRELATION_WEIGHT, - emlib.MDL_IMED + #emlib.MDL_COST, + #emlib.MDL_WEIGHT, + #emlib.MDL_CORRELATION_IDX, + #emlib.MDL_CORRELATION_MASK, + #emlib.MDL_CORRELATION_WEIGHT, + #emlib.MDL_IMED ] - """ # Fill readSetOfParticles( self._getOutputParticlesMdFilename(), - particleSet - #extraLabels=EXTRA_LABELS + particles, + extraLabels=EXTRA_LABELS ) - particleSet.setSamplingRate(self._getSamplingRate()) + particles.setSamplingRate(self._getSamplingRate()) + self._insertChild('outputParticles', particles) + + def updateClass(cls: Class3D): + clsId = cls.getObjId() + representative = volumes[clsId] + cls.setRepresentative(representative) + + classes3d = self._createSetOfClasses3D(particles) + classes3d.classifyItems(updateClassCallback=updateClass) # Define the output - self._defineOutputs(outputParticles=particleSet) - self._defineSourceRelation(self.inputParticles.get(), particleSet) + self._defineOutputs(outputClasses=classes3d) + self._defineSourceRelation(self.inputParticles, classes3d) + self._defineSourceRelation(self.inputVolumes, classes3d) - return particleSet + return classes3d - def _createOutputFsc(self): - fsc = FSC() + def _createOutputVolumes(self): + volumes = self._createSetOfVolumes() + volumes.setSamplingRate(self._getSamplingRate()) - # Load from metadata - cls = 0 # TODO adapt for multiple FSCs - fsc.loadFromMd( - self._getOutputFscFilename(cls), - emlib.MDL_RESOLUTION_FREQ, - emlib.MDL_RESOLUTION_FRC - ) + for cls in range(self._getClassCount()): + volume=Volume(objId=cls+1) + + # Fill + volume.setFileName(self._getOutputVolumeFilename(cls)) + volume.setHalfMaps([ + self._getOutputHalfVolumeFilename(cls, 1), + self._getOutputHalfVolumeFilename(cls, 2), + ]) + + volumes.append(volume) + + # Define the output + self._defineOutputs(outputVolumes=volumes) + self._defineSourceRelation(self.inputParticles, volumes) + self._defineSourceRelation(self.inputVolumes, volumes) + + return volumes + + def _createOutputFscs(self): + fscs = self._createSetOfFSCs() + + for cls in range(self._getClassCount()): + fsc = FSC(objId=cls+1) + + # Load from metadata + fsc.loadFromMd( + self._getOutputFscFilename(cls), + emlib.MDL_RESOLUTION_FREQ, + emlib.MDL_RESOLUTION_FRC + ) + + fscs.append(fsc) # Define the output - self._defineOutputs(outputFSC=fsc) - self._defineSourceRelation(self.inputParticles.get(), fsc) + self._defineOutputs(outputFSC=fscs) + self._defineSourceRelation(self.inputParticles, fscs) + self._defineSourceRelation(self.inputVolumes, fscs) - return fsc + return fscs def _runMdUtils(self, args): self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) From 2a7528dc56babd9387150c570fc2acc185a1d37c Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 2 Feb 2023 16:08:08 +0100 Subject: [PATCH 020/104] Renamed gallery.doc to gallery.xmd --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 2e98086ff..77787c128 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -484,7 +484,7 @@ def _getClassGalleryStackFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'gallery.mrcs') def _getGalleryMdFilename(self, iteration: int): - return self._getIterationPath(iteration, 'gallery.doc') + return self._getIterationPath(iteration, 'gallery.xmd') def _getWeightsFilename(self, iteration: int): return self._getIterationPath(iteration, 'weights.mrc') From 328b16e578650567e51b47fb8d97a48efba3deb7 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 2 Feb 2023 17:17:40 +0100 Subject: [PATCH 021/104] Bugfix: Disordered iteration and class --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 77787c128..8739b851c 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -417,11 +417,11 @@ def createOutputStep(self): ) createLink( - self._getAverageVolumeFilename(cls, lastIteration), # TODO replace with post-processed volume + self._getAverageVolumeFilename(lastIteration, cls), # TODO replace with post-processed volume self._getOutputVolumeFilename(cls) ) createLink( - self._getFscFilename(cls, lastIteration), + self._getFscFilename(lastIteration, cls), self._getOutputFscFilename(cls) ) From 75ca73cd1e209b1d1fb2f735c765a17938aabe48 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 3 Feb 2023 12:04:08 +0100 Subject: [PATCH 022/104] Added particle filtering --- .../protocol_reconstruct_swiftres.py | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 8739b851c..59cc30fbe 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -72,6 +72,7 @@ def _defineParams(self, form: Form): form.addParam('angularSampling', FloatParam, label="Angular sampling (º)", default=5.0) form.addParam('shiftCount', IntParam, label="Shifts", default=9) form.addParam('maxShift', FloatParam, label="Maximum shift (%)", default=10.0) + form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50) form.addSection(label='Compute') form.addParam('databaseRecipe', StringParam, label='Database recipe', @@ -131,6 +132,7 @@ def _insertReconstructSteps(self, iteration: int, cls: int, prerequisites): compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, cls, prerequisites=[selectAlignmentStepId]) compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[compareAnglesStepId]) computeWeightsStepId = self._insertFunctionStep('computeWeightsStep', iteration, cls, prerequisites=[compareReprojectionStepId]) + #filterByWeightsStepId = self._insertFunctionStep('filterByWeightsStep', iteration, cls, prerequisites=[computeWeightsStepId]) splitStepId = self._insertFunctionStep('splitStep', iteration, cls, prerequisites=[computeWeightsStepId]) reconstructStepId1 = self._insertFunctionStep('reconstructStep', iteration, cls, 1, prerequisites=[splitStepId]) reconstructStepId2 = self._insertFunctionStep('reconstructStep', iteration, cls, 2, prerequisites=[splitStepId]) @@ -139,11 +141,8 @@ def _insertReconstructSteps(self, iteration: int, cls: int, prerequisites): return [computeFscStepId, averageVolumeStepId] def _insertPostProcessSteps(self, iteration: int, cls: int, prerequisites): - """ - filterVolumeStepId = self._insertFunctionStep('filterVolumeStep', iteration, prerequisites=prerequisites) + filterVolumeStepId = self._insertFunctionStep('filterVolumeStep', iteration, cls, prerequisites=prerequisites) return [filterVolumeStepId] - """ - return prerequisites #--------------------------- STEPS functions -------------------------------------------- def convertInputStep(self): @@ -287,21 +286,27 @@ def compareReprojectionStep(self, iteration: int, cls: int): self.runJob('xmipp_angular_continuous_assign2', args, numberOfMpi=self.numberOfMpi.get()) def computeWeightsStep(self, iteration: int, cls: int): - """ args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] args += ['--fill', 'weight', 'constant', '0.0'] self._runMdUtils(args) args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] args += ['--operate', 'modify_values', 'weight=corrIdx*corrWeight*corrMask'] self._runMdUtils(args) - """ - + + def filterByWeightsStep(self, iteration: int, cls: int): args = [] args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] - args += ['--operate', 'rename_column', 'weightContinuous2 weight'] + args += ['-o', self._getFilteredClassAlignmentMdFilename(iteration, cls)] + args += ['--operate', 'percentile', 'weight', 'weight'] + self._runMdUtils(args) + + args = [] + args += ['-i', self._getFilteredClassAlignmentMdFilename(iteration, cls)] + args += ['-o', self._getFilteredClassAlignmentMdFilename(iteration, cls)] + args += ['--query', 'select', 'weight>=%f' % self._getReconstructPercentile()] self._runMdUtils(args) def splitStep(self, iteration: int, cls: int): @@ -316,7 +321,7 @@ def reconstructStep(self, iteration: int, cls: int, half: int): args += ['-i', self._getClassAlignmentHalfMdFilename(iteration, cls, half)] args += ['-o', self._getHalfVolumeFilename(iteration, cls, half)] args += ['--sym', self.symmetryGroup.get()] - args += ['--weight'] + args += ['--weight'] # TODO determine if used # Determine the execution parameters numberOfMpi = self.numberOfMpi.get() @@ -358,7 +363,6 @@ def reconstructStep(self, iteration: int, cls: int, half: int): # Run self.runJob(reconstructProgram, args, numberOfMpi=numberOfMpi) - def computeFscStep(self, iteration: int, cls: int): args = [] args += ['--ref', self._getHalfVolumeFilename(iteration, cls, 1)] @@ -417,7 +421,7 @@ def createOutputStep(self): ) createLink( - self._getAverageVolumeFilename(lastIteration, cls), # TODO replace with post-processed volume + self._getFilteredVolumeFilename(lastIteration, cls), self._getOutputVolumeFilename(cls) ) createLink( @@ -438,6 +442,9 @@ def _getClassCount(self) -> int: def _getMaxShift(self) -> float: return float(self.maxShift) / 100.0 + def _getReconstructPercentile(self) -> float: + return 1.0 - (float(self.reconstructPercentage) / 100.0) + def _getSamplingRate(self) -> float: return float(self.inputParticles.get().getSamplingRate()) @@ -473,7 +480,7 @@ def _getInputVolumeFilename(self, cls: int): def _getIterationInputVolumeFilename(self, iteration: int, cls: int): if iteration > 0: - return self._getAverageVolumeFilename(iteration-1, cls) # TODO replace with post processed volume + return self._getFilteredVolumeFilename(iteration-1, cls) else: return self._getInputVolumeFilename(cls) @@ -507,6 +514,9 @@ def _getAngleDiffMdFilename(self, iteration: int, cls: int): def _getClassAlignmentMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'aligned.xmd') + def _getFilteredClassAlignmentMdFilename(self, iteration: int, cls: int): + return self._getClassPath(iteration, cls, 'well_aligned.xmd') + def _getClassAlignmentHalfMdFilename(self, iteration: int, cls: int, half: int): return self._getClassPath(iteration, cls, 'aligned%06d.xmd' % half) From f7aa7556e26d20ee92b9654ad14f5eaa7332e440 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 6 Feb 2023 10:03:56 +0100 Subject: [PATCH 023/104] Added help --- .../protocol_reconstruct_swiftres.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 59cc30fbe..c7a0efc81 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -56,29 +56,39 @@ def _defineParams(self, form: Form): form.addSection(label='Input') form.addParam('inputParticles', PointerParam, label="Particles", important=True, - pointerClass='SetOfParticles') + pointerClass='SetOfParticles', + help='Input particle set') form.addParam('inputVolumes', MultiPointerParam, label="Initial volumes", important=True, - pointerClass='Volume') + pointerClass='Volume', + help='Provide a volume for each class of interest') form.addParam('symmetryGroup', StringParam, default="c1", label='Symmetry group', help='If no symmetry is present, give c1') form.addSection(label='Refinement') form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) - form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0) + form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0, + help='Resolution limit at the first iteration of the refinement') form.addParam('nextResolutionCriterion', FloatParam, label="FSC criterion", default=0.5, expertLevel=LEVEL_ADVANCED, help='The resolution of the reconstruction is defined as the inverse of the frequency at which '\ 'the FSC drops below this value. Typical values are 0.143 and 0.5' ) - form.addParam('angularSampling', FloatParam, label="Angular sampling (º)", default=5.0) - form.addParam('shiftCount', IntParam, label="Shifts", default=9) - form.addParam('maxShift', FloatParam, label="Maximum shift (%)", default=10.0) - form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50) + form.addParam('angularSampling', FloatParam, label="Angular sampling (º)", default=5.0, + help='Angular sampling in the first iteration') + form.addParam('shiftCount', IntParam, label="Shifts", default=9, + help='Number of shifts considered in each axis') + form.addParam('maxShift', FloatParam, label="Maximum shift (%)", default=10.0, + help='Maximum shift of the particle in terms of its size') + form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, + help='Percentage of best particles used for reconstruction') form.addSection(label='Compute') form.addParam('databaseRecipe', StringParam, label='Database recipe', - default='OPQ48_192,IVF32768,PQ48' ) + default='OPQ48_192,IVF32768,PQ48', expertLevel=LEVEL_ADVANCED, + help='FAISS database structure. Please refer to ' + 'https://github.com/facebookresearch/faiss/wiki/The-index-factory') form.addParam('databaseTrainingSetSize', IntParam, label='Database training set size', - default=int(2e6) ) + default=int(2e6), + help='Number of data-augmented particles to used when training the database') form.addParallelSection(threads=1, mpi=8) From 7c309b8f61589f7f2dc7c0d178a2912d9939706a Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 9 Feb 2023 10:53:07 +0100 Subject: [PATCH 024/104] Added maximum DB size parameter --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index c7a0efc81..845a0e658 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -89,6 +89,10 @@ def _defineParams(self, form: Form): form.addParam('databaseTrainingSetSize', IntParam, label='Database training set size', default=int(2e6), help='Number of data-augmented particles to used when training the database') + form.addParam('databaseMaximumSize', IntParam, label='Database size limit', + default=int(2e6), + help='Maximum number of elements that can be stored in the database ' + 'before performing an alignment and flush') form.addParallelSection(threads=1, mpi=8) @@ -254,6 +258,7 @@ def alignStep(self, iteration: int): args += ['--method', 'fourier'] args += ['--dropna'] args += ['--batch', batchSize] + args += ['--max_size', self.databaseMaximumSize] if self.useGpu: args += ['--gpu', 0] # TODO select @@ -267,14 +272,14 @@ def selectAlignmentStep(self, iteration: int, cls: int): args += ['-o', self._getClassAlignmentMdFilename(iteration, cls)] args += ['--query', 'select', 'ref3d==%d' % (cls+1)] self._runMdUtils(args) - - def compareAnglesStep(self, iteration: int, cls): + args = [] args += ['-i', self._getInputParticleMdFilename()] args += ['-o', self._getInputIntersectionMdFilename(iteration, cls)] args += ['--set', 'intersection', self._getClassAlignmentMdFilename(iteration, cls), 'itemId'] self._runMdUtils(args) - + + def compareAnglesStep(self, iteration: int, cls): args = [] args += ['--ang1', self._getClassAlignmentMdFilename(iteration, cls)] args += ['--ang2', self._getInputIntersectionMdFilename(iteration, cls)] @@ -409,14 +414,14 @@ def filterVolumeStep(self, iteration: int, cls: int): def createOutputStep(self): lastIteration = int(self.numberOfIterations) - 1 - # Keep only the image and id from the input particle set + # Rename the wiener filtered image column args = [] args += ['-i', self._getAlignmentMdFilename(lastIteration)] args += ['-o', self._getOutputParticlesMdFilename()] args += ['--operate', 'rename_column', 'image image1'] self._runMdUtils(args) - # Add the rest from the last alignment + # Add the input image column args = [] args += ['-i', self._getOutputParticlesMdFilename()] args += ['--set', 'join', self._getInputParticleMdFilename(), 'itemId'] From 77b7d9dac58d4b1deca643694555e369a0d926f3 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 10 Feb 2023 10:12:11 +0100 Subject: [PATCH 025/104] Worked with centered particles --- .../protocol_reconstruct_swiftres.py | 151 ++++++++++++++---- 1 file changed, 117 insertions(+), 34 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 845a0e658..20b04f657 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -65,7 +65,7 @@ def _defineParams(self, form: Form): label='Symmetry group', help='If no symmetry is present, give c1') - form.addSection(label='Refinement') + form.addSection(label='Global refinement') form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0, help='Resolution limit at the first iteration of the refinement') @@ -111,7 +111,6 @@ def _insertAllSteps(self): self._insertFunctionStep('createOutputStep', prerequisites=lastIds) def _insertIterationSteps(self, iteration: int, prerequisites): - setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, prerequisites=prerequisites) projectIds = self._insertProjectSteps(iteration, prerequisites=[setupIterationStepId]) alignIds = self._insertAlignmentSteps(iteration, prerequisites=projectIds) @@ -121,8 +120,9 @@ def _insertIterationSteps(self, iteration: int, prerequisites): reconstructIds = self._insertReconstructSteps(iteration, cls, prerequisites=alignIds) postProcessIds = self._insertPostProcessSteps(iteration, cls, prerequisites=reconstructIds) ids += postProcessIds - - return ids + + mergeAlignmentsId = self._insertFunctionStep('mergeAlignmentsStep', iteration, prerequisites=ids) + return [mergeAlignmentsId] def _insertProjectSteps(self, iteration: int, prerequisites): # Project all volumes @@ -178,12 +178,34 @@ def correctCtfStep(self): args += ['--phase_flipped'] self.runJob('xmipp_ctf_correct_wiener2d', args) - + def setupIterationStep(self, iteration: int): makePath(self._getIterationPath(iteration)) for cls in range(self._getClassCount()): makePath(self._getClassPath(iteration, cls)) + + if iteration > 0: + # Fill missing particles from the previous alignment with + # The input particles. In order to use union, columns must + # match, so join columns prior to doing the union + args = [] + args += ['-i', self._getWienerParticleMdFilename()] + args += ['-o', self._getIterationInputParticleMdFilename(iteration)] + args += ['--set', 'join', self._getCenteredAlignmentMdFilename(iteration-1), 'itemId'] + self._runMdUtils(args) + + args = [] + args += ['-i', self._getCenteredAlignmentMdFilename(iteration-1)] + args += ['-o', self._getIterationInputParticleMdFilename(iteration)] + args += ['--set', 'union', self._getIterationInputParticleMdFilename(iteration), 'itemId'] + self._runMdUtils(args) + + else: + createLink( + self._getWienerParticleMdFilename(), + self._getIterationInputParticleMdFilename(iteration) + ) def projectVolumeStep(self, iteration: int, cls: int): args = [] @@ -191,7 +213,6 @@ def projectVolumeStep(self, iteration: int, cls: int): args += ['-o', self._getClassGalleryStackFilename(iteration, cls)] args += ['--sampling_rate', self.angularSampling] args += ['--sym', self.symmetryGroup] - self.runJob('xmipp_angular_project_library', args) args = [] @@ -244,9 +265,10 @@ def alignStep(self, iteration: int): batchSize = 1024 nRotations = round(360 / float(self.angularSampling)) nShift = self.shiftCount - + + # Perform the alignment args = [] - args += ['-i', self._getWienerParticleMdFilename()] + args += ['-i', self._getIterationInputParticleMdFilename(iteration)] args += ['-o', self._getAlignmentMdFilename(iteration)] args += ['-r', self._getGalleryMdFilename(iteration)] args += ['--index', self._getTrainingIndexFilename(iteration)] @@ -258,82 +280,116 @@ def alignStep(self, iteration: int): args += ['--method', 'fourier'] args += ['--dropna'] args += ['--batch', batchSize] - args += ['--max_size', self.databaseMaximumSize] + #args += ['--max_size', self.databaseMaximumSize] # TODO uncomment when added if self.useGpu: args += ['--gpu', 0] # TODO select env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) + + # Accumulate shifts in the input set + if iteration > 0: + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--operate', 'modify_values', 'shiftX=shiftX+shiftX2;shiftY=shiftY+shiftY2'] + self._runMdUtils(args) + + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--operate', 'modify_values', 'imageOriginal=ifnull(imageOriginal, image)'] + self._runMdUtils(args) + + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--operate', 'drop_column', 'image shiftX2 shiftY2'] + self._runMdUtils(args) + + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--operate', 'rename_column', 'imageOriginal image'] + self._runMdUtils(args) def selectAlignmentStep(self, iteration: int, cls: int): args = [] args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['-o', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['-o', self._getReconstructionMdFilename(iteration, cls)] args += ['--query', 'select', 'ref3d==%d' % (cls+1)] self._runMdUtils(args) args = [] - args += ['-i', self._getInputParticleMdFilename()] + args += ['-i', self._getIterationInputParticleMdFilename(iteration)] # TODO use the unshifted ones args += ['-o', self._getInputIntersectionMdFilename(iteration, cls)] - args += ['--set', 'intersection', self._getClassAlignmentMdFilename(iteration, cls), 'itemId'] + args += ['--set', 'intersection', self._getReconstructionMdFilename(iteration, cls), 'itemId'] self._runMdUtils(args) def compareAnglesStep(self, iteration: int, cls): args = [] - args += ['--ang1', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['--ang1', self._getReconstructionMdFilename(iteration, cls)] args += ['--ang2', self._getInputIntersectionMdFilename(iteration, cls)] args += ['--oroot', self._getAngleDiffOutputRoot(iteration, cls)] args += ['--sym', self.symmetryGroup] self.runJob('xmipp_angular_distance', args, numberOfMpi=1) - args = [] - args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] - args += ['--set', 'join', self._getAngleDiffMdFilename(iteration, cls)] - self._runMdUtils(args) + #args = [] + #args += ['-i', self._getReconstructionMdFilename(iteration, cls)] + #args += ['--set', 'join', self._getAngleDiffMdFilename(iteration, cls), 'itemId'] + #self._runMdUtils(args) def compareReprojectionStep(self, iteration: int, cls: int): + # Save shifts for future use + args = [] + args += ['-i', self._getReconstructionMdFilename(iteration, cls)] + args += ['--fill', 'shiftX2 shiftY2', 'constant', 0.0] + self._runMdUtils(args) + + args = [] + args += ['-i', self._getReconstructionMdFilename(iteration, cls)] + args += ['--operate', 'modify_values', 'shiftX2=shiftX;shiftY2=shiftY'] + self._runMdUtils(args) + args = [] - args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['-i', self._getReconstructionMdFilename(iteration, cls)] + args += ['-o', self._getReconstructionStackFilename(iteration, cls)] args += ['--ref', self._getIterationInputVolumeFilename(iteration, cls)] args += ['--ignoreCTF'] # As we're using wiener corrected images - args += ['--doNotWriteStack'] # Do not undo shifts + #args += ['--doNotWriteStack'] # Do not undo shifts self.runJob('xmipp_angular_continuous_assign2', args, numberOfMpi=self.numberOfMpi.get()) def computeWeightsStep(self, iteration: int, cls: int): args = [] - args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['-i', self._getReconstructionMdFilename(iteration, cls)] args += ['--fill', 'weight', 'constant', '0.0'] self._runMdUtils(args) args = [] - args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['-i', self._getReconstructionMdFilename(iteration, cls)] args += ['--operate', 'modify_values', 'weight=corrIdx*corrWeight*corrMask'] self._runMdUtils(args) def filterByWeightsStep(self, iteration: int, cls: int): args = [] - args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] - args += ['-o', self._getFilteredClassAlignmentMdFilename(iteration, cls)] + args += ['-i', self._getReconstructionMdFilename(iteration, cls)] + args += ['-o', self._getFilteredReconstructionMdFilename(iteration, cls)] args += ['--operate', 'percentile', 'weight', 'weight'] self._runMdUtils(args) args = [] - args += ['-i', self._getFilteredClassAlignmentMdFilename(iteration, cls)] - args += ['-o', self._getFilteredClassAlignmentMdFilename(iteration, cls)] + args += ['-i', self._getFilteredReconstructionMdFilename(iteration, cls)] + args += ['-o', self._getFilteredReconstructionMdFilename(iteration, cls)] args += ['--query', 'select', 'weight>=%f' % self._getReconstructPercentile()] self._runMdUtils(args) def splitStep(self, iteration: int, cls: int): args = [] - args += ['-i', self._getClassAlignmentMdFilename(iteration, cls)] + args += ['-i', self._getReconstructionMdFilename(iteration, cls)] args += ['-n', 2] self.runJob('xmipp_metadata_split', args, numberOfMpi=1) def reconstructStep(self, iteration: int, cls: int, half: int): args = [] - args += ['-i', self._getClassAlignmentHalfMdFilename(iteration, cls, half)] + args += ['-i', self._getReconstructionHalfMdFilename(iteration, cls, half)] args += ['-o', self._getHalfVolumeFilename(iteration, cls, half)] args += ['--sym', self.symmetryGroup.get()] args += ['--weight'] # TODO determine if used @@ -411,6 +467,20 @@ def filterVolumeStep(self, iteration: int, cls: int): self.runJob('xmipp_transform_filter', args, numberOfMpi=1) + def mergeAlignmentsStep(self, iteration: int): + # Copy the first alignment + copyFile( + self._getReconstructionMdFilename(iteration, 0), + self._getCenteredAlignmentMdFilename(iteration) + ) + + # Merge subsequent alignments + for cls in range(1, self._getClassCount()): + args = [] + args += ['-i', self._getCenteredAlignmentMdFilename()] + args += ['--set', 'union', self._getReconstructionMdFilename(iteration, cls)] + self._runMdUtils(args) + def createOutputStep(self): lastIteration = int(self.numberOfIterations) - 1 @@ -465,10 +535,14 @@ def _getSamplingRate(self) -> float: def _getIterationResolutionLimit(self, iteration: int) -> float: if iteration > 0: - mdFsc = emlib.MetaData(self._getFscFilename(iteration-1)) - sampling = self._getSamplingRate() - threshold = float(self.nextResolutionCriterion) - return self._computeResolution(mdFsc, sampling, threshold) + res = 0.0 + for cls in range(self._getClassCount()): + mdFsc = emlib.MetaData(self._getFscFilename(iteration-1, cls)) + sampling = self._getSamplingRate() + threshold = float(self.nextResolutionCriterion) + res += self._computeResolution(mdFsc, sampling, threshold) + + return res / self._getClassCount() else: return float(self.initialResolution) @@ -498,6 +572,9 @@ def _getIterationInputVolumeFilename(self, iteration: int, cls: int): return self._getFilteredVolumeFilename(iteration-1, cls) else: return self._getInputVolumeFilename(cls) + + def _getIterationInputParticleMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'input_particles.xmd') def _getClassGalleryMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'gallery.doc') @@ -517,6 +594,9 @@ def _getTrainingIndexFilename(self, iteration: int): def _getAlignmentMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'aligned.xmd') + def _getCenteredAlignmentMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'aligned_centered.xmd') + def _getInputIntersectionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'input_intersection.xmd') @@ -526,13 +606,16 @@ def _getAngleDiffOutputRoot(self, iteration: int, cls: int, suffix=''): def _getAngleDiffMdFilename(self, iteration: int, cls: int): return self._getAngleDiffOutputRoot(iteration, cls, '.xmd') - def _getClassAlignmentMdFilename(self, iteration: int, cls: int): + def _getReconstructionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'aligned.xmd') + + def _getReconstructionStackFilename(self, iteration: int, cls: int): + return self._getClassPath(iteration, cls, 'aligned.mrcs') - def _getFilteredClassAlignmentMdFilename(self, iteration: int, cls: int): + def _getFilteredReconstructionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'well_aligned.xmd') - def _getClassAlignmentHalfMdFilename(self, iteration: int, cls: int, half: int): + def _getReconstructionHalfMdFilename(self, iteration: int, cls: int, half: int): return self._getClassPath(iteration, cls, 'aligned%06d.xmd' % half) def _getHalfVolumeFilename(self, iteration: int, cls: int, half: int): From 19382add3331e6621ccd3e61b2b4253f7dd05321 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 13 Feb 2023 08:51:07 +0100 Subject: [PATCH 026/104] Considered moving particle centering to program --- .../protocol_reconstruct_swiftres.py | 71 ++----------------- 1 file changed, 7 insertions(+), 64 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 20b04f657..590369c8b 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -121,8 +121,7 @@ def _insertIterationSteps(self, iteration: int, prerequisites): postProcessIds = self._insertPostProcessSteps(iteration, cls, prerequisites=reconstructIds) ids += postProcessIds - mergeAlignmentsId = self._insertFunctionStep('mergeAlignmentsStep', iteration, prerequisites=ids) - return [mergeAlignmentsId] + return ids def _insertProjectSteps(self, iteration: int, prerequisites): # Project all volumes @@ -196,12 +195,13 @@ def setupIterationStep(self, iteration: int): self._runMdUtils(args) args = [] - args += ['-i', self._getCenteredAlignmentMdFilename(iteration-1)] + args += ['-i', self._getAlignmentMdFilename(iteration-1)] args += ['-o', self._getIterationInputParticleMdFilename(iteration)] args += ['--set', 'union', self._getIterationInputParticleMdFilename(iteration), 'itemId'] self._runMdUtils(args) else: + # For the first iteration, simply use the input particles. createLink( self._getWienerParticleMdFilename(), self._getIterationInputParticleMdFilename(iteration) @@ -220,26 +220,6 @@ def projectVolumeStep(self, iteration: int, cls: int): args += ['--fill', 'ref3d', 'constant', cls+1] self._runMdUtils(args) - def mergeGalleriesStep(self, iteration): - # Copy the first gallery - copyFile( - self._getClassGalleryMdFilename(iteration, 0), - self._getGalleryMdFilename(iteration) - ) - - # Merge subsequent galleries - for cls in range(1, self._getClassCount()): - args = [] - args += ['-i', self._getGalleryMdFilename(iteration)] - args += ['--set', 'union', self._getClassGalleryMdFilename(iteration, cls)] - self._runMdUtils(args) - - # Reindex - args = [] - args += ['-i', self._getGalleryMdFilename(iteration)] - args += ['--fill', 'ref', 'lineal', 1, 1] - self._runMdUtils(args) - def trainDatabaseStep(self, iteration: int): trainingSize = int(self.databaseTrainingSetSize) recipe = self.databaseRecipe @@ -283,32 +263,12 @@ def alignStep(self, iteration: int): #args += ['--max_size', self.databaseMaximumSize] # TODO uncomment when added if self.useGpu: args += ['--gpu', 0] # TODO select + if iteration > 0: + pass # TODO add centering env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) - - # Accumulate shifts in the input set - if iteration > 0: - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--operate', 'modify_values', 'shiftX=shiftX+shiftX2;shiftY=shiftY+shiftY2'] - self._runMdUtils(args) - - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--operate', 'modify_values', 'imageOriginal=ifnull(imageOriginal, image)'] - self._runMdUtils(args) - - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--operate', 'drop_column', 'image shiftX2 shiftY2'] - self._runMdUtils(args) - - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--operate', 'rename_column', 'imageOriginal image'] - self._runMdUtils(args) def selectAlignmentStep(self, iteration: int, cls: int): args = [] @@ -318,7 +278,7 @@ def selectAlignmentStep(self, iteration: int, cls: int): self._runMdUtils(args) args = [] - args += ['-i', self._getIterationInputParticleMdFilename(iteration)] # TODO use the unshifted ones + args += ['-i', self._getIterationInputParticleMdFilename(iteration)] args += ['-o', self._getInputIntersectionMdFilename(iteration, cls)] args += ['--set', 'intersection', self._getReconstructionMdFilename(iteration, cls), 'itemId'] self._runMdUtils(args) @@ -337,23 +297,12 @@ def compareAnglesStep(self, iteration: int, cls): #self._runMdUtils(args) def compareReprojectionStep(self, iteration: int, cls: int): - # Save shifts for future use - args = [] - args += ['-i', self._getReconstructionMdFilename(iteration, cls)] - args += ['--fill', 'shiftX2 shiftY2', 'constant', 0.0] - self._runMdUtils(args) - - args = [] - args += ['-i', self._getReconstructionMdFilename(iteration, cls)] - args += ['--operate', 'modify_values', 'shiftX2=shiftX;shiftY2=shiftY'] - self._runMdUtils(args) - args = [] args += ['-i', self._getReconstructionMdFilename(iteration, cls)] args += ['-o', self._getReconstructionStackFilename(iteration, cls)] args += ['--ref', self._getIterationInputVolumeFilename(iteration, cls)] args += ['--ignoreCTF'] # As we're using wiener corrected images - #args += ['--doNotWriteStack'] # Do not undo shifts + args += ['--doNotWriteStack'] # Do not undo shifts self.runJob('xmipp_angular_continuous_assign2', args, numberOfMpi=self.numberOfMpi.get()) def computeWeightsStep(self, iteration: int, cls: int): @@ -594,9 +543,6 @@ def _getTrainingIndexFilename(self, iteration: int): def _getAlignmentMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'aligned.xmd') - def _getCenteredAlignmentMdFilename(self, iteration: int): - return self._getIterationPath(iteration, 'aligned_centered.xmd') - def _getInputIntersectionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'input_intersection.xmd') @@ -609,9 +555,6 @@ def _getAngleDiffMdFilename(self, iteration: int, cls: int): def _getReconstructionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'aligned.xmd') - def _getReconstructionStackFilename(self, iteration: int, cls: int): - return self._getClassPath(iteration, cls, 'aligned.mrcs') - def _getFilteredReconstructionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'well_aligned.xmd') From 5816fd00b558799d6c1daad3ae7be4ba605136bd Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 13 Feb 2023 13:03:31 +0100 Subject: [PATCH 027/104] Continued removing unused parameters --- .../protocol_reconstruct_swiftres.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 590369c8b..75a23d74c 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -68,7 +68,9 @@ def _defineParams(self, form: Form): form.addSection(label='Global refinement') form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0, - help='Resolution limit at the first iteration of the refinement') + help='Image comparison resolution limit at the first iteration of the refinement') + form.addParam('maximumResolution', FloatParam, label="Maximum resolution (A)", default=8.0, + help='Image comparison resolution limit of the refinement') form.addParam('nextResolutionCriterion', FloatParam, label="FSC criterion", default=0.5, expertLevel=LEVEL_ADVANCED, help='The resolution of the reconstruction is defined as the inverse of the frequency at which '\ 'the FSC drops below this value. Typical values are 0.143 and 0.5' ) @@ -191,7 +193,7 @@ def setupIterationStep(self, iteration: int): args = [] args += ['-i', self._getWienerParticleMdFilename()] args += ['-o', self._getIterationInputParticleMdFilename(iteration)] - args += ['--set', 'join', self._getCenteredAlignmentMdFilename(iteration-1), 'itemId'] + args += ['--set', 'join', self._getAlignmentMdFilename(iteration-1), 'itemId'] self._runMdUtils(args) args = [] @@ -260,11 +262,11 @@ def alignStep(self, iteration: int): args += ['--method', 'fourier'] args += ['--dropna'] args += ['--batch', batchSize] - #args += ['--max_size', self.databaseMaximumSize] # TODO uncomment when added + args += ['--max_size', self.databaseMaximumSize] if self.useGpu: args += ['--gpu', 0] # TODO select if iteration > 0: - pass # TODO add centering + args += ['--local_shift', '--local_psi'] env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it @@ -299,7 +301,6 @@ def compareAnglesStep(self, iteration: int, cls): def compareReprojectionStep(self, iteration: int, cls: int): args = [] args += ['-i', self._getReconstructionMdFilename(iteration, cls)] - args += ['-o', self._getReconstructionStackFilename(iteration, cls)] args += ['--ref', self._getIterationInputVolumeFilename(iteration, cls)] args += ['--ignoreCTF'] # As we're using wiener corrected images args += ['--doNotWriteStack'] # Do not undo shifts @@ -308,7 +309,7 @@ def compareReprojectionStep(self, iteration: int, cls: int): def computeWeightsStep(self, iteration: int, cls: int): args = [] args += ['-i', self._getReconstructionMdFilename(iteration, cls)] - args += ['--fill', 'weight', 'constant', '0.0'] + args += ['--fill', 'weight', 'constant', 0.0] self._runMdUtils(args) args = [] @@ -485,15 +486,17 @@ def _getSamplingRate(self) -> float: def _getIterationResolutionLimit(self, iteration: int) -> float: if iteration > 0: res = 0.0 + threshold = float(self.nextResolutionCriterion) for cls in range(self._getClassCount()): mdFsc = emlib.MetaData(self._getFscFilename(iteration-1, cls)) sampling = self._getSamplingRate() - threshold = float(self.nextResolutionCriterion) res += self._computeResolution(mdFsc, sampling, threshold) - return res / self._getClassCount() + res /= self._getClassCount() else: - return float(self.initialResolution) + res = float(self.initialResolution) + + return max(res, float(self.maximumResolution)) def _getIterationDigitalFrequencyLimit(self, iteration: int) -> float: return self._getSamplingRate() / self._getIterationResolutionLimit(iteration) From ebba207e318ce028dfd52b293530bae38ca2d094 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 14 Feb 2023 10:13:44 +0100 Subject: [PATCH 028/104] Worked on automating parameters --- .../protocol_reconstruct_swiftres.py | 122 +++++++++++++----- 1 file changed, 91 insertions(+), 31 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 75a23d74c..ac61e106e 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -35,6 +35,8 @@ import xmipp3 from xmipp3.convert import writeSetOfParticles, readSetOfParticles +import math + class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): _label = 'swiftres' _conda_env = 'xmipp_torch' @@ -78,7 +80,7 @@ def _defineParams(self, form: Form): help='Angular sampling in the first iteration') form.addParam('shiftCount', IntParam, label="Shifts", default=9, help='Number of shifts considered in each axis') - form.addParam('maxShift', FloatParam, label="Maximum shift (%)", default=10.0, + form.addParam('initialMaxShift', FloatParam, label="Maximum shift (%)", default=10.0, help='Maximum shift of the particle in terms of its size') form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, help='Percentage of best particles used for reconstruction') @@ -95,6 +97,9 @@ def _defineParams(self, form: Form): default=int(2e6), help='Maximum number of elements that can be stored in the database ' 'before performing an alignment and flush') + form.addParam('batchSize', IntParam, label='Batch size', + default=1024, + help='Batch size used when processing') form.addParallelSection(threads=1, mpi=8) @@ -202,19 +207,51 @@ def setupIterationStep(self, iteration: int): args += ['--set', 'union', self._getIterationInputParticleMdFilename(iteration), 'itemId'] self._runMdUtils(args) + resolutionLimit = self._computeIterationResolution(iteration-1) + else: # For the first iteration, simply use the input particles. createLink( self._getWienerParticleMdFilename(), self._getIterationInputParticleMdFilename(iteration) ) - + + resolutionLimit = float(self.initialResolution) + + maxFrequency = self._getSamplingRate() / resolutionLimit + maxShift = self._getIterationMaxShift(iteration) + shiftStep = self._computeShiftStep(maxFrequency) + angleStep = self._computeAngleStep(maxFrequency, 160) #TODO + + # Write to metadata + md = emlib.MetaData() + id = md.addObject() + md.setValue(emlib.MDL_RESOLUTION_FREQ, resolutionLimit, id) + md.setValue(emlib.MDL_RESOLUTION_FREQREAL, maxFrequency, id) + md.setValue(emlib.MDL_SHIFT_DIFF2, maxShift, id) + md.setValue(emlib.MDL_SHIFT_DIFF, shiftStep, id) + md.setValue(emlib.MDL_ANGLE_DIFF, angleStep, id) + md.write(self._getIterationParametersFilename(iteration)) + + def projectVolumeStep(self, iteration: int, cls: int): + md = emlib.MetaData(self._getIterationParametersFilename(iteration)) + angleStep = md.getValue(emlib.MDL_ANGLE_DIFF, 1) + perturb = math.sin(math.radians(angleStep)) / 4 + args = [] args += ['-i', self._getIterationInputVolumeFilename(iteration, cls)] args += ['-o', self._getClassGalleryStackFilename(iteration, cls)] - args += ['--sampling_rate', self.angularSampling] + args += ['--sampling_rate', angleStep] + args += ['--perturb', perturb] args += ['--sym', self.symmetryGroup] + + if False: # TODO + #if iteration > 0: + args += ['--compute_neighbors'] + args += ['--angular_distance', -1] + args += ['--experimental_images', self._getIterationInputParticleMdFilename(iteration)] + self.runJob('xmipp_angular_project_library', args) args = [] @@ -226,15 +263,20 @@ def trainDatabaseStep(self, iteration: int): trainingSize = int(self.databaseTrainingSetSize) recipe = self.databaseRecipe + md = emlib.MetaData(self._getIterationParametersFilename(iteration)) + maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) + maxShift = md.getValue(emlib.MDL_SHIFT_DIFF2, 1) + args = [] args += ['-i', self._getGalleryMdFilename(iteration)] args += ['-o', self._getTrainingIndexFilename(iteration)] args += ['--recipe', recipe] #args += ['--weights', self._getWeightsFilename(iteration)] - args += ['--max_shift', self._getMaxShift()] - args += ['--max_frequency', self._getIterationDigitalFrequencyLimit(iteration)] + args += ['--max_shift', maxShift] + args += ['--max_frequency', maxFrequency] args += ['--method', 'fourier'] args += ['--training', trainingSize] + args += ['--batch', self.batchSize] args += ['--scratch', self._getTrainingScratchFilename()] if self.useGpu: args += ['--gpu', 0] # TODO select @@ -244,9 +286,11 @@ def trainDatabaseStep(self, iteration: int): self.runJob('xmipp_train_database', args, numberOfMpi=1, env=env) def alignStep(self, iteration: int): - batchSize = 1024 - nRotations = round(360 / float(self.angularSampling)) - nShift = self.shiftCount + md = emlib.MetaData(self._getIterationParametersFilename(iteration)) + maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) + maxShift = md.getValue(emlib.MDL_SHIFT_DIFF2, 1) + nShift = round((2*maxShift) / md.getValue(emlib.MDL_SHIFT_DIFF, 1)) + 1 + nRotations = round(360 / md.getValue(emlib.MDL_ANGLE_DIFF, 1)) # Perform the alignment args = [] @@ -255,13 +299,13 @@ def alignStep(self, iteration: int): args += ['-r', self._getGalleryMdFilename(iteration)] args += ['--index', self._getTrainingIndexFilename(iteration)] #args += ['--weights', self._getWeightsFilename(iteration)] - args += ['--max_shift', self._getMaxShift()] + args += ['--max_shift', maxShift] args += ['--rotations', nRotations] args += ['--shifts', nShift] - args += ['--max_frequency', self._getIterationDigitalFrequencyLimit(iteration)] + args += ['--max_frequency', maxFrequency] args += ['--method', 'fourier'] args += ['--dropna'] - args += ['--batch', batchSize] + args += ['--batch', self.batchSize] args += ['--max_size', self.databaseMaximumSize] if self.useGpu: args += ['--gpu', 0] # TODO select @@ -474,33 +518,13 @@ def createOutputStep(self): def _getClassCount(self) -> int: return len(self.inputVolumes) - def _getMaxShift(self) -> float: - return float(self.maxShift) / 100.0 - def _getReconstructPercentile(self) -> float: return 1.0 - (float(self.reconstructPercentage) / 100.0) def _getSamplingRate(self) -> float: return float(self.inputParticles.get().getSamplingRate()) - def _getIterationResolutionLimit(self, iteration: int) -> float: - if iteration > 0: - res = 0.0 - threshold = float(self.nextResolutionCriterion) - for cls in range(self._getClassCount()): - mdFsc = emlib.MetaData(self._getFscFilename(iteration-1, cls)) - sampling = self._getSamplingRate() - res += self._computeResolution(mdFsc, sampling, threshold) - - res /= self._getClassCount() - else: - res = float(self.initialResolution) - return max(res, float(self.maximumResolution)) - - def _getIterationDigitalFrequencyLimit(self, iteration: int) -> float: - return self._getSamplingRate() / self._getIterationResolutionLimit(iteration) - def _getIterationPath(self, iteration: int, *paths): return self._getExtraPath('iteration_%04d' % iteration, *paths) @@ -519,6 +543,9 @@ def _getWienerParticleStackFilename(self): def _getInputVolumeFilename(self, cls: int): return self.inputVolumes[cls].get().getFileName() + def _getIterationParametersFilename(self, iteration: int): + return self._getIterationPath(iteration, 'params.xmd') + def _getIterationInputVolumeFilename(self, iteration: int, cls: int): if iteration > 0: return self._getFilteredVolumeFilename(iteration-1, cls) @@ -603,6 +630,39 @@ def _computeResolution(self, mdFsc, Ts, threshold): return resolution + def _computeIterationResolution(self, iteration: int) -> float: + res = 0.0 + threshold = float(self.nextResolutionCriterion) + for cls in range(self._getClassCount()): + mdFsc = emlib.MetaData(self._getFscFilename(iteration, cls)) + sampling = self._getSamplingRate() + res += self._computeResolution(mdFsc, sampling, threshold) + + res /= self._getClassCount() + + return max(res, float(self.maximumResolution)) + + def _computeAngleStep(self, maxFrequency: float, size: int) -> float: + # At the alignment resolution limit, determine the + # angle between to neighboring Fourier coefficients + c = maxFrequency*size # Cos: radius + s = 1.0 # Sin: 1 coefficient + angle = math.atan2(s, c) + + # The angular error is at most half of the sampling + # Therefore use the double angular sampling + angle *= 2 + + return math.degrees(angle) + + def _computeShiftStep(self, digital_freq: float, eps: float = 0.5) -> float: + # Assuming that in Nyquist (0.5) we are able to + # detect a shift of eps pixels + return (0.5 / digital_freq) * eps + + def _getIterationMaxShift(self, iteration: int) -> float: + return float(self.initialMaxShift) / math.pow(math.e, iteration) + def _createOutputClasses3D(self, volumes: SetOfVolumes): particles = self._createSetOfParticles() From 492776bbab15298aeccee3833cf5d842d6e4b1a5 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 14 Feb 2023 10:24:01 +0100 Subject: [PATCH 029/104] Fixed mergeAlignmentStep for future use --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index ac61e106e..364a300ab 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -465,14 +465,14 @@ def mergeAlignmentsStep(self, iteration: int): # Copy the first alignment copyFile( self._getReconstructionMdFilename(iteration, 0), - self._getCenteredAlignmentMdFilename(iteration) + self._getAlignmentMdFilename(iteration) ) # Merge subsequent alignments for cls in range(1, self._getClassCount()): args = [] - args += ['-i', self._getCenteredAlignmentMdFilename()] - args += ['--set', 'union', self._getReconstructionMdFilename(iteration, cls)] + args += ['-i', self._getAlignmentMdFilename()] + args += ['--set', 'union', self._getReconstructionMdFilename(iteration, cls)] #TODO consider using union_all self._runMdUtils(args) def createOutputStep(self): From d81d58aa633bb6b2c9782a365d3172e4bad10d02 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 14 Feb 2023 10:34:24 +0100 Subject: [PATCH 030/104] Added maxPsi --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 364a300ab..1a1f25fd0 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -219,6 +219,7 @@ def setupIterationStep(self, iteration: int): resolutionLimit = float(self.initialResolution) maxFrequency = self._getSamplingRate() / resolutionLimit + maxPsi = self._getIterationMaxPsi(iteration) maxShift = self._getIterationMaxShift(iteration) shiftStep = self._computeShiftStep(maxFrequency) angleStep = self._computeAngleStep(maxFrequency, 160) #TODO @@ -228,7 +229,9 @@ def setupIterationStep(self, iteration: int): id = md.addObject() md.setValue(emlib.MDL_RESOLUTION_FREQ, resolutionLimit, id) md.setValue(emlib.MDL_RESOLUTION_FREQREAL, maxFrequency, id) - md.setValue(emlib.MDL_SHIFT_DIFF2, maxShift, id) + md.setValue(emlib.MDL_ANGLE_PSI, maxPsi, id) + md.setValue(emlib.MDL_SHIFT_X, maxShift, id) + md.setValue(emlib.MDL_SHIFT_Y, maxShift, id) md.setValue(emlib.MDL_SHIFT_DIFF, shiftStep, id) md.setValue(emlib.MDL_ANGLE_DIFF, angleStep, id) md.write(self._getIterationParametersFilename(iteration)) @@ -265,7 +268,8 @@ def trainDatabaseStep(self, iteration: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) - maxShift = md.getValue(emlib.MDL_SHIFT_DIFF2, 1) + maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) + maxShift = md.getValue(emlib.MDL_SHIFT_X, 1) args = [] args += ['-i', self._getGalleryMdFilename(iteration)] @@ -288,7 +292,8 @@ def trainDatabaseStep(self, iteration: int): def alignStep(self, iteration: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) - maxShift = md.getValue(emlib.MDL_SHIFT_DIFF2, 1) + maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) + maxShift = md.getValue(emlib.MDL_SHIFT_X, 1) nShift = round((2*maxShift) / md.getValue(emlib.MDL_SHIFT_DIFF, 1)) + 1 nRotations = round(360 / md.getValue(emlib.MDL_ANGLE_DIFF, 1)) @@ -660,6 +665,9 @@ def _computeShiftStep(self, digital_freq: float, eps: float = 0.5) -> float: # detect a shift of eps pixels return (0.5 / digital_freq) * eps + def _getIterationMaxPsi(self, iteration: int) -> float: + return 180.0 / math.pow(math.e, iteration) + def _getIterationMaxShift(self, iteration: int) -> float: return float(self.initialMaxShift) / math.pow(math.e, iteration) From 27eef64d70e2605d6f1c42e8b40f390023085dbb Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 16 Feb 2023 10:46:11 +0100 Subject: [PATCH 031/104] Re-added mergeGalleriesStep --- .../protocol_reconstruct_swiftres.py | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 1a1f25fd0..4a0930071 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -57,13 +57,16 @@ def _defineParams(self, form: Form): help="Add a list of GPU devices that can be used") form.addSection(label='Input') - form.addParam('inputParticles', PointerParam, label="Particles", important=True, + form.addParam('inputParticles', PointerParam, label='Particles', important=True, pointerClass='SetOfParticles', help='Input particle set') - form.addParam('inputVolumes', MultiPointerParam, label="Initial volumes", important=True, - pointerClass='Volume', + form.addParam('considerInputAlignment', BooleanParam, label='Consider previous alignment', + default=True, + help='Consider the alignment of input particles') + form.addParam('inputVolumes', MultiPointerParam, label='Initial volumes', important=True, + pointerClass='Volume', minNumObjects=1, help='Provide a volume for each class of interest') - form.addParam('symmetryGroup', StringParam, default="c1", + form.addParam('symmetryGroup', StringParam, default='c1', label='Symmetry group', help='If no symmetry is present, give c1') @@ -80,7 +83,7 @@ def _defineParams(self, form: Form): help='Angular sampling in the first iteration') form.addParam('shiftCount', IntParam, label="Shifts", default=9, help='Number of shifts considered in each axis') - form.addParam('initialMaxShift', FloatParam, label="Maximum shift (%)", default=10.0, + form.addParam('initialMaxShift', FloatParam, label='Maximum shift (%)', default=10.0, help='Maximum shift of the particle in terms of its size') form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, help='Percentage of best particles used for reconstruction') @@ -249,8 +252,7 @@ def projectVolumeStep(self, iteration: int, cls: int): args += ['--perturb', perturb] args += ['--sym', self.symmetryGroup] - if False: # TODO - #if iteration > 0: + if False: # TODO speak with coss args += ['--compute_neighbors'] args += ['--angular_distance', -1] args += ['--experimental_images', self._getIterationInputParticleMdFilename(iteration)] @@ -262,6 +264,26 @@ def projectVolumeStep(self, iteration: int, cls: int): args += ['--fill', 'ref3d', 'constant', cls+1] self._runMdUtils(args) + def mergeGalleriesStep(self, iteration): + # Copy the first gallery + copyFile( + self._getClassGalleryMdFilename(iteration, 0), + self._getGalleryMdFilename(iteration) + ) + + # Merge subsequent galleries + for cls in range(1, self._getClassCount()): + args = [] + args += ['-i', self._getGalleryMdFilename(iteration)] + args += ['--set', 'union', self._getClassGalleryMdFilename(iteration, cls)] + self._runMdUtils(args) + + # Reindex + args = [] + args += ['-i', self._getGalleryMdFilename(iteration)] + args += ['--fill', 'ref', 'lineal', 1, 1] + self._runMdUtils(args) + def trainDatabaseStep(self, iteration: int): trainingSize = int(self.databaseTrainingSetSize) recipe = self.databaseRecipe @@ -754,5 +776,3 @@ def _createOutputFscs(self): def _runMdUtils(self, args): self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) - - \ No newline at end of file From deb5f3e9343ad88f5328aa98a369afb2c3b98e89 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 16 Feb 2023 12:23:51 +0100 Subject: [PATCH 032/104] Added merging of metadata --- .../protocol_reconstruct_swiftres.py | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 4a0930071..684b6628c 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -35,6 +35,7 @@ import xmipp3 from xmipp3.convert import writeSetOfParticles, readSetOfParticles +from typing import Iterable import math class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): @@ -76,13 +77,10 @@ def _defineParams(self, form: Form): help='Image comparison resolution limit at the first iteration of the refinement') form.addParam('maximumResolution', FloatParam, label="Maximum resolution (A)", default=8.0, help='Image comparison resolution limit of the refinement') - form.addParam('nextResolutionCriterion', FloatParam, label="FSC criterion", default=0.5, expertLevel=LEVEL_ADVANCED, + form.addParam('nextResolutionCriterion', FloatParam, label="FSC criterion", default=0.5, + expertLevel=LEVEL_ADVANCED, help='The resolution of the reconstruction is defined as the inverse of the frequency at which '\ 'the FSC drops below this value. Typical values are 0.143 and 0.5' ) - form.addParam('angularSampling', FloatParam, label="Angular sampling (º)", default=5.0, - help='Angular sampling in the first iteration') - form.addParam('shiftCount', IntParam, label="Shifts", default=9, - help='Number of shifts considered in each axis') form.addParam('initialMaxShift', FloatParam, label='Maximum shift (%)', default=10.0, help='Maximum shift of the particle in terms of its size') form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, @@ -275,7 +273,7 @@ def mergeGalleriesStep(self, iteration): for cls in range(1, self._getClassCount()): args = [] args += ['-i', self._getGalleryMdFilename(iteration)] - args += ['--set', 'union', self._getClassGalleryMdFilename(iteration, cls)] + args += ['--set', 'union_all', self._getClassGalleryMdFilename(iteration, cls)] self._runMdUtils(args) # Reindex @@ -299,6 +297,7 @@ def trainDatabaseStep(self, iteration: int): args += ['--recipe', recipe] #args += ['--weights', self._getWeightsFilename(iteration)] args += ['--max_shift', maxShift] + args += ['--max_psi', maxPsi] args += ['--max_frequency', maxFrequency] args += ['--method', 'fourier'] args += ['--training', trainingSize] @@ -327,6 +326,7 @@ def alignStep(self, iteration: int): args += ['--index', self._getTrainingIndexFilename(iteration)] #args += ['--weights', self._getWeightsFilename(iteration)] args += ['--max_shift', maxShift] + args += ['--max_psi', maxPsi] args += ['--rotations', nRotations] args += ['--shifts', nShift] args += ['--max_frequency', maxFrequency] @@ -360,13 +360,13 @@ def compareAnglesStep(self, iteration: int, cls): args = [] args += ['--ang1', self._getReconstructionMdFilename(iteration, cls)] args += ['--ang2', self._getInputIntersectionMdFilename(iteration, cls)] - args += ['--oroot', self._getAngleDiffOutputRoot(iteration, cls)] + args += ['--oroot', self._getReconstructionAngleDiffOutputRoot(iteration, cls)] args += ['--sym', self.symmetryGroup] self.runJob('xmipp_angular_distance', args, numberOfMpi=1) #args = [] #args += ['-i', self._getReconstructionMdFilename(iteration, cls)] - #args += ['--set', 'join', self._getAngleDiffMdFilename(iteration, cls), 'itemId'] + #args += ['--set', 'join', self._getReconstructionAngleDiffMdFilename(iteration, cls), 'itemId'] #self._runMdUtils(args) def compareReprojectionStep(self, iteration: int, cls: int): @@ -498,9 +498,23 @@ def mergeAlignmentsStep(self, iteration: int): # Merge subsequent alignments for cls in range(1, self._getClassCount()): args = [] - args += ['-i', self._getAlignmentMdFilename()] + args += ['-i', self._getAlignmentMdFilename(iteration)] args += ['--set', 'union', self._getReconstructionMdFilename(iteration, cls)] #TODO consider using union_all self._runMdUtils(args) + + def mergeAnglesStep(self, iteration: int): + # Copy the first alignment + copyFile( + self._getReconstructionAngleDiffMdFilename(iteration, 0), + self._getAngleDiffMdFilename(iteration) + ) + + # Merge subsequent alignments + for cls in range(1, self._getClassCount()): + args = [] + args += ['-i', self._getAngleDiffMdFilename(iteration)] + args += ['--set', 'union', self._getReconstructionAngleDiffMdFilename(iteration, cls)] #TODO consider using union_all + self._runMdUtils(args) def createOutputStep(self): lastIteration = int(self.numberOfIterations) - 1 @@ -603,11 +617,14 @@ def _getAlignmentMdFilename(self, iteration: int): def _getInputIntersectionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'input_intersection.xmd') - def _getAngleDiffOutputRoot(self, iteration: int, cls: int, suffix=''): + def _getReconstructionAngleDiffOutputRoot(self, iteration: int, cls: int, suffix=''): return self._getClassPath(iteration, cls, 'angles'+suffix) - def _getAngleDiffMdFilename(self, iteration: int, cls: int): - return self._getAngleDiffOutputRoot(iteration, cls, '.xmd') + def _getReconstructionAngleDiffMdFilename(self, iteration: int, cls: int): + return self._getReconstructionAngleDiffOutputRoot(iteration, cls, '.xmd') + + def _getAngleDiffMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'angles.xmd') def _getReconstructionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'aligned.xmd') @@ -774,5 +791,18 @@ def _createOutputFscs(self): return fscs + def _mergeMetadata(self, src: Iterable[str], dst: str): + it = iter(src) + + # Copy the first alignment + copyFile(next(it), dst) + + # Merge subsequent alignments + for s in it: + args = [] + args += ['-i', dst] + args += ['--set', 'union_all', s] + self._runMdUtils(args) + def _runMdUtils(self, args): self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) From ccd65df99759939dbabebfb2e8f2909c03d2a4eb Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 17 Feb 2023 14:39:38 +0100 Subject: [PATCH 033/104] Updated to use merge function --- .../protocol_reconstruct_swiftres.py | 36 ++++--------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 684b6628c..5d1bd7681 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -263,18 +263,10 @@ def projectVolumeStep(self, iteration: int, cls: int): self._runMdUtils(args) def mergeGalleriesStep(self, iteration): - # Copy the first gallery - copyFile( - self._getClassGalleryMdFilename(iteration, 0), + self._mergeMetadata( + map(lambda cls : self._getClassGalleryMdFilename(iteration, cls), range(self._getClassCount())), self._getGalleryMdFilename(iteration) ) - - # Merge subsequent galleries - for cls in range(1, self._getClassCount()): - args = [] - args += ['-i', self._getGalleryMdFilename(iteration)] - args += ['--set', 'union_all', self._getClassGalleryMdFilename(iteration, cls)] - self._runMdUtils(args) # Reindex args = [] @@ -489,32 +481,16 @@ def filterVolumeStep(self, iteration: int, cls: int): self.runJob('xmipp_transform_filter', args, numberOfMpi=1) def mergeAlignmentsStep(self, iteration: int): - # Copy the first alignment - copyFile( - self._getReconstructionMdFilename(iteration, 0), + self._mergeMetadata( + map(lambda cls : self._getReconstructionMdFilename(iteration, cls), range(self._getClassCount())), self._getAlignmentMdFilename(iteration) ) - - # Merge subsequent alignments - for cls in range(1, self._getClassCount()): - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--set', 'union', self._getReconstructionMdFilename(iteration, cls)] #TODO consider using union_all - self._runMdUtils(args) def mergeAnglesStep(self, iteration: int): - # Copy the first alignment - copyFile( - self._getReconstructionAngleDiffMdFilename(iteration, 0), + self._mergeMetadata( + map(lambda cls : self._getReconstructionAngleDiffMdFilename(iteration, cls), range(self._getClassCount())), self._getAngleDiffMdFilename(iteration) ) - - # Merge subsequent alignments - for cls in range(1, self._getClassCount()): - args = [] - args += ['-i', self._getAngleDiffMdFilename(iteration)] - args += ['--set', 'union', self._getReconstructionAngleDiffMdFilename(iteration, cls)] #TODO consider using union_all - self._runMdUtils(args) def createOutputStep(self): lastIteration = int(self.numberOfIterations) - 1 From 727dd620cdca2012ec90345d4b2d83596e2d0e81 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 23 Feb 2023 19:04:16 +0100 Subject: [PATCH 034/104] Updated parameters --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 5d1bd7681..071da79f5 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -95,7 +95,7 @@ def _defineParams(self, form: Form): default=int(2e6), help='Number of data-augmented particles to used when training the database') form.addParam('databaseMaximumSize', IntParam, label='Database size limit', - default=int(2e6), + default=int(10e6), help='Maximum number of elements that can be stored in the database ' 'before performing an alignment and flush') form.addParam('batchSize', IntParam, label='Batch size', @@ -237,7 +237,6 @@ def setupIterationStep(self, iteration: int): md.setValue(emlib.MDL_ANGLE_DIFF, angleStep, id) md.write(self._getIterationParametersFilename(iteration)) - def projectVolumeStep(self, iteration: int, cls: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) angleStep = md.getValue(emlib.MDL_ANGLE_DIFF, 1) @@ -281,7 +280,7 @@ def trainDatabaseStep(self, iteration: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) - maxShift = md.getValue(emlib.MDL_SHIFT_X, 1) + maxShift = md.getValue(emlib.MDL_SHIFT_X, 1) / 100 args = [] args += ['-i', self._getGalleryMdFilename(iteration)] @@ -296,7 +295,7 @@ def trainDatabaseStep(self, iteration: int): args += ['--batch', self.batchSize] args += ['--scratch', self._getTrainingScratchFilename()] if self.useGpu: - args += ['--gpu', 0] # TODO select + args += ['--device', 'cuda:0'] # TODO select env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it @@ -306,7 +305,7 @@ def alignStep(self, iteration: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) - maxShift = md.getValue(emlib.MDL_SHIFT_X, 1) + maxShift = md.getValue(emlib.MDL_SHIFT_X, 1) / 100 nShift = round((2*maxShift) / md.getValue(emlib.MDL_SHIFT_DIFF, 1)) + 1 nRotations = round(360 / md.getValue(emlib.MDL_ANGLE_DIFF, 1)) @@ -327,8 +326,8 @@ def alignStep(self, iteration: int): args += ['--batch', self.batchSize] args += ['--max_size', self.databaseMaximumSize] if self.useGpu: - args += ['--gpu', 0] # TODO select - if iteration > 0: + args += ['--device', 'cuda:0'] # TODO select + if iteration > 0 or self.considerInputAlignment: args += ['--local_shift', '--local_psi'] env = self.getCondaEnv() From 951d7e4b274213889d25539ee17c7e597d3e8455 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 23 Feb 2023 19:18:17 +0100 Subject: [PATCH 035/104] Bugfix --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 071da79f5..9ecee5bea 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -303,10 +303,12 @@ def trainDatabaseStep(self, iteration: int): def alignStep(self, iteration: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) + imageSize = 160 # TODO determine maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) maxShift = md.getValue(emlib.MDL_SHIFT_X, 1) / 100 - nShift = round((2*maxShift) / md.getValue(emlib.MDL_SHIFT_DIFF, 1)) + 1 + maxShiftPx = imageSize * maxShift + nShift = round((2*maxShiftPx) / md.getValue(emlib.MDL_SHIFT_DIFF, 1)) + 1 nRotations = round(360 / md.getValue(emlib.MDL_ANGLE_DIFF, 1)) # Perform the alignment From 217d83e8f510914d9e2d62a0e4ce50b8495eb8fa Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 23 Feb 2023 19:21:23 +0100 Subject: [PATCH 036/104] Changed max shift to be in pixels --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 9ecee5bea..8f412020b 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -81,7 +81,7 @@ def _defineParams(self, form: Form): expertLevel=LEVEL_ADVANCED, help='The resolution of the reconstruction is defined as the inverse of the frequency at which '\ 'the FSC drops below this value. Typical values are 0.143 and 0.5' ) - form.addParam('initialMaxShift', FloatParam, label='Maximum shift (%)', default=10.0, + form.addParam('initialMaxShift', FloatParam, label='Maximum shift (px)', default=16.0, help='Maximum shift of the particle in terms of its size') form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, help='Percentage of best particles used for reconstruction') @@ -278,9 +278,11 @@ def trainDatabaseStep(self, iteration: int): recipe = self.databaseRecipe md = emlib.MetaData(self._getIterationParametersFilename(iteration)) + imageSize = 160 # TODO determine maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) - maxShift = md.getValue(emlib.MDL_SHIFT_X, 1) / 100 + maxShiftPx = md.getValue(emlib.MDL_SHIFT_X, 1) + maxShift = maxShiftPx / imageSize args = [] args += ['-i', self._getGalleryMdFilename(iteration)] @@ -306,8 +308,8 @@ def alignStep(self, iteration: int): imageSize = 160 # TODO determine maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) - maxShift = md.getValue(emlib.MDL_SHIFT_X, 1) / 100 - maxShiftPx = imageSize * maxShift + maxShiftPx = md.getValue(emlib.MDL_SHIFT_X, 1) + maxShift = maxShiftPx / imageSize nShift = round((2*maxShiftPx) / md.getValue(emlib.MDL_SHIFT_DIFF, 1)) + 1 nRotations = round(360 / md.getValue(emlib.MDL_ANGLE_DIFF, 1)) From 59e615758ca355fdc50a9ad8549e4edd3b0fd147 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 24 Feb 2023 09:56:54 +0100 Subject: [PATCH 037/104] Created viewer --- .../protocol_reconstruct_swiftres.py | 22 ++- xmipp3/viewers/__init__.py | 1 + xmipp3/viewers/viewer_reconstruct_swiftres.py | 166 ++++++++++++++++++ 3 files changed, 182 insertions(+), 7 deletions(-) create mode 100644 xmipp3/viewers/viewer_reconstruct_swiftres.py diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 8f412020b..f5180495a 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -81,8 +81,11 @@ def _defineParams(self, form: Form): expertLevel=LEVEL_ADVANCED, help='The resolution of the reconstruction is defined as the inverse of the frequency at which '\ 'the FSC drops below this value. Typical values are 0.143 and 0.5' ) + form.addParam('initialMaxPsi', FloatParam, label='Maximum psi (deg)', default=180.0, + expertLevel=LEVEL_ADVANCED, + help='Maximum psi parameter of the particles') form.addParam('initialMaxShift', FloatParam, label='Maximum shift (px)', default=16.0, - help='Maximum shift of the particle in terms of its size') + help='Maximum shift of the particle in pixels') form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, help='Percentage of best particles used for reconstruction') @@ -113,7 +116,7 @@ def _insertAllSteps(self): correctCtfStepId = self._insertFunctionStep('correctCtfStep', prerequisites=[convertInputStepId]) lastIds = [correctCtfStepId] - for i in range(int(self.numberOfIterations)): + for i in range(self._getIterationCount()): lastIds = self._insertIterationSteps(i, prerequisites=lastIds) self._insertFunctionStep('createOutputStep', prerequisites=lastIds) @@ -220,7 +223,8 @@ def setupIterationStep(self, iteration: int): resolutionLimit = float(self.initialResolution) maxFrequency = self._getSamplingRate() / resolutionLimit - maxPsi = self._getIterationMaxPsi(iteration) + #maxPsi = self._getIterationMaxPsi(iteration) + maxPsi = 180.0 maxShift = self._getIterationMaxShift(iteration) shiftStep = self._computeShiftStep(maxFrequency) angleStep = self._computeAngleStep(maxFrequency, 160) #TODO @@ -332,7 +336,8 @@ def alignStep(self, iteration: int): if self.useGpu: args += ['--device', 'cuda:0'] # TODO select if iteration > 0 or self.considerInputAlignment: - args += ['--local_shift', '--local_psi'] + #args += ['--local_shift', '--local_psi'] + args += ['--local_shift'] env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it @@ -496,7 +501,7 @@ def mergeAnglesStep(self, iteration: int): ) def createOutputStep(self): - lastIteration = int(self.numberOfIterations) - 1 + lastIteration = self._getIterationCount() - 1 # Rename the wiener filtered image column args = [] @@ -535,6 +540,9 @@ def createOutputStep(self): #--------------------------- UTILS functions -------------------------------------------- + def _getIterationCount(self) -> int: + return int(self.numberOfIterations) + def _getClassCount(self) -> int: return len(self.inputVolumes) @@ -684,10 +692,10 @@ def _computeShiftStep(self, digital_freq: float, eps: float = 0.5) -> float: return (0.5 / digital_freq) * eps def _getIterationMaxPsi(self, iteration: int) -> float: - return 180.0 / math.pow(math.e, iteration) + return float(self.initialMaxPsi) / math.pow(2.0, iteration) def _getIterationMaxShift(self, iteration: int) -> float: - return float(self.initialMaxShift) / math.pow(math.e, iteration) + return float(self.initialMaxShift) / math.pow(2.0, iteration) def _createOutputClasses3D(self, volumes: SetOfVolumes): particles = self._createSetOfParticles() diff --git a/xmipp3/viewers/__init__.py b/xmipp3/viewers/__init__.py index be27b95eb..3241182a2 100644 --- a/xmipp3/viewers/__init__.py +++ b/xmipp3/viewers/__init__.py @@ -47,6 +47,7 @@ from .viewer_validate_overfitting import XmippValidateOverfittingViewer from .viewer_volume_strain import XmippVolumeStrainViewer from .viewer_reconstruct_highres import XmippReconstructHighResViewer +from .viewer_reconstruct_swiftres import XmippReconstructSwiftresViewer from .viewer_solid_angles import SolidAnglesViewer from .viewer_consensus_classes3D import XmippConsensusClasses3DViewer from .viewer_classes3D import XmippClasses3DViewer diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py new file mode 100644 index 000000000..f35f2495c --- /dev/null +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -0,0 +1,166 @@ +# ************************************************************************** +# * +# * Authors: Oier Lauzirika Zarrabeitia (oierlauzi@bizkaia.eu) +# * +# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC +# * +# * This program is free software; you can redistribute it and/or modify +# * it under the terms of the GNU General Public License as published by +# * the Free Software Foundation; either version 2 of the License, or +# * (at your option) any later version. +# * +# * This program is distributed in the hope that it will be useful, +# * but WITHOUT ANY WARRANTY; without even the implied warranty of +# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# * GNU General Public License for more details. +# * +# * You should have received a copy of the GNU General Public License +# * along with this program; if not, write to the Free Software +# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +# * 02111-1307 USA +# * +# * All comments concerning this program package may be sent to the +# * e-mail address 'scipion@cnb.csic.es' +# * +# ************************************************************************** + +import os +import itertools +import numpy as np +import matplotlib.pyplot as plt + +from pyworkflow.viewer import ProtocolViewer, DESKTOP_TKINTER, WEB_DJANGO +from pyworkflow.protocol.params import LabelParam, IntParam, FloatParam + +from pwem import emlib +from pwem.viewers.views import DataView + +from xmipp3.protocols.protocol_reconstruct_swiftres import XmippProtReconstructSwiftres + + + +class XmippReconstructSwiftresViewer(ProtocolViewer): + """ Wrapper to visualize different type of data objects + with the Xmipp program xmipp_showj + """ + _label = 'viewer swiftres' + _environments = [DESKTOP_TKINTER, WEB_DJANGO] + _targets = [XmippProtReconstructSwiftres] + + # --------------------------- DEFINE param functions ----------------------- + def _defineParams(self, form): + form.addSection(label='FSC') + form.addParam('showIterationFsc', IntParam, label='Display iteration FSC', + default=0 ) + form.addParam('showClassFsc', IntParam, label='Display class FSC', + default=0) + form.addParam('showFscCutoff', FloatParam, label='Display FSC cutoff', + default=0.143) + + def _getVisualizeDict(self): + return { + 'showIterationFsc': self._showIterationFsc, + 'showClassFsc': self._showClassFsc, + 'showFscCutoff': self._showFscCutoff + } + + + # --------------------------- UTILS functions ------------------------------ + def _getIterationCount(self) -> int: + return self.protocol._getIterationCount() + + def _getClassCount(self) -> int: + return self.protocol._getClassCount() + + def _getSamplingRate(self) -> float: + return self.protocol._getSamplingRate() + + def _getFscFilename(self, iteration: int, cls: int): + return self.protocol._getFscFilename(iteration, cls) + + def _readFsc(self, filename: str) -> np.ndarray: + fscMd = emlib.MetaData(filename) + + values = [] + for objId in fscMd: + freq = fscMd.getValue(emlib.MDL_RESOLUTION_FREQ, objId) + fsc = fscMd.getValue(emlib.MDL_RESOLUTION_FRC, objId) + values.append((freq, fsc)) + + return np.array(values) + + def _readFscCutoff(self, cls: int, cutoff: float) -> np.ndarray: + samplingRate = self._getSamplingRate() + + values = [] + for it in itertools.count(): + fscFn = self._getFscFilename(it, cls) + if os.path.exists(fscFn): + fscMd = emlib.MetaData(fscFn) + values.append(self.protocol._computeResolution( + fscMd, + samplingRate, + cutoff + )) + else: + break + + return np.array(values) + + # ---------------------------- SHOW functions ------------------------------ + def _showIterationFsc(self, e): + fig, ax = plt.subplots() + + it = int(self.showIterationFsc) + for cls in itertools.count(): + fscFn = self._getFscFilename(it, cls) + if os.path.exists(fscFn): + fsc = self._readFsc(fscFn) + label = f'Class {cls}' + ax.plot(fsc[:,0], fsc[:,1], label=label) + else: + break + + ax.set_title('Class FSC') + ax.set_xlabel('Resolution (1/A)') + ax.set_ylabel('FSC') + ax.legend() + + return [fig] + + def _showClassFsc(self, e): + fig, ax = plt.subplots() + + cls = int(self.showClassFsc) + for it in itertools.count(): + fscFn = self._getFscFilename(it, cls) + if os.path.exists(fscFn): + fsc = self._readFsc(fscFn) + label = f'Iteration {it}' + ax.plot(fsc[:,0], fsc[:,1], label=label) + else: + break + + ax.set_title('Class FSC') + ax.set_xlabel('Resolution (1/A)') + ax.set_ylabel('FSC') + ax.legend() + + return [fig] + + def _showFscCutoff(self, e): + fig, ax = plt.subplots() + + cutoff = float(self.showFscCutoff) + for cls in range(self._getClassCount()): + y = self._readFscCutoff(cls, cutoff) + x = np.arange(len(y)) + label = f'Class {cls}' + ax.plot(x, y, label=label) + + ax.set_title('Resolution') + ax.set_xlabel('Iteration') + ax.set_ylabel('Resolution (A)') + ax.legend() + + return [fig] \ No newline at end of file From 81d4c34257d620437a735ac92b674ef88106794b Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 24 Feb 2023 18:17:08 +0100 Subject: [PATCH 038/104] Worked in the viewer --- .../protocol_reconstruct_swiftres.py | 8 +- xmipp3/viewers/viewer_reconstruct_swiftres.py | 154 +++++++++++++++--- 2 files changed, 137 insertions(+), 25 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index f5180495a..994e0c5a4 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -223,8 +223,7 @@ def setupIterationStep(self, iteration: int): resolutionLimit = float(self.initialResolution) maxFrequency = self._getSamplingRate() / resolutionLimit - #maxPsi = self._getIterationMaxPsi(iteration) - maxPsi = 180.0 + maxPsi = self._getIterationMaxPsi(iteration) maxShift = self._getIterationMaxShift(iteration) shiftStep = self._computeShiftStep(maxFrequency) angleStep = self._computeAngleStep(maxFrequency, 160) #TODO @@ -336,8 +335,7 @@ def alignStep(self, iteration: int): if self.useGpu: args += ['--device', 'cuda:0'] # TODO select if iteration > 0 or self.considerInputAlignment: - #args += ['--local_shift', '--local_psi'] - args += ['--local_shift'] + args += ['--local_shift', '--local_psi'] env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it @@ -693,9 +691,11 @@ def _computeShiftStep(self, digital_freq: float, eps: float = 0.5) -> float: def _getIterationMaxPsi(self, iteration: int) -> float: return float(self.initialMaxPsi) / math.pow(2.0, iteration) + #return float(self.initialMaxPsi) def _getIterationMaxShift(self, iteration: int) -> float: return float(self.initialMaxShift) / math.pow(2.0, iteration) + #return float(self.initialMaxShift) def _createOutputClasses3D(self, volumes: SetOfVolumes): particles = self._createSetOfParticles() diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index f35f2495c..67033eb80 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -24,9 +24,12 @@ # * # ************************************************************************** +from typing import Tuple, Iterator import os import itertools +import collections import numpy as np +import matplotlib.collections import matplotlib.pyplot as plt from pyworkflow.viewer import ProtocolViewer, DESKTOP_TKINTER, WEB_DJANGO @@ -57,11 +60,17 @@ def _defineParams(self, form): form.addParam('showFscCutoff', FloatParam, label='Display FSC cutoff', default=0.143) + form.addSection(label='Classification') + form.addParam('showClassMigration', LabelParam, label='Display class migration diagram', + default=0.143) + def _getVisualizeDict(self): return { 'showIterationFsc': self._showIterationFsc, 'showClassFsc': self._showClassFsc, - 'showFscCutoff': self._showFscCutoff + 'showFscCutoff': self._showFscCutoff, + + 'showClassMigration': self._showClassMigration } @@ -75,12 +84,13 @@ def _getClassCount(self) -> int: def _getSamplingRate(self) -> float: return self.protocol._getSamplingRate() + def _getAlignmentMdFilename(self, iteration: int): + return self.protocol._getAlignmentMdFilename(iteration) + def _getFscFilename(self, iteration: int, cls: int): return self.protocol._getFscFilename(iteration, cls) - def _readFsc(self, filename: str) -> np.ndarray: - fscMd = emlib.MetaData(filename) - + def _readFsc(self, fscMd: emlib.MetaData) -> np.ndarray: values = [] for objId in fscMd: freq = fscMd.getValue(emlib.MDL_RESOLUTION_FREQ, objId) @@ -89,6 +99,33 @@ def _readFsc(self, filename: str) -> np.ndarray: return np.array(values) + def _iterAlignmentMdFilenames(self): + for it in itertools.count(): + alignmentFn = self._getAlignmentMdFilename(it) + if os.path.exists(alignmentFn): + yield alignmentFn + + else: + break + + def _iterFscMdIterationFilenames(self, it: int): + for cls in itertools.count(): + fscFn = self._getFscFilename(it, cls) + if os.path.exists(fscFn): + yield fscFn + + else: + break + + def _iterFscMdClassFilenames(self, cls: int): + for it in itertools.count(): + fscFn = self._getFscFilename(it, cls) + if os.path.exists(fscFn): + yield fscFn + + else: + break + def _readFscCutoff(self, cls: int, cutoff: float) -> np.ndarray: samplingRate = self._getSamplingRate() @@ -107,19 +144,78 @@ def _readFscCutoff(self, cls: int, cutoff: float) -> np.ndarray: return np.array(values) + def _computeClassMigrations(self, srcMd: emlib.MetaData, dstMd: emlib.MetaData) -> dict: + result = {} + + for objId in srcMd: + srcClassId = srcMd.getValue(emlib.MDL_REF3D, objId) + dstClassId = dstMd.getValue(emlib.MDL_REF3D, objId) + + key = (srcClassId, dstClassId) + if key in result: + result[key] += 1 + else: + result[key] = 1 + + return result + + def _computeIterationClassMigrationSegments(self, migrations: dict) -> Tuple[np.ndarray, np.ndarray]: + segments = np.empty((len(migrations), 2)) + counts = np.array(list(migrations.values())) + + # Write the y values form migrations + for i, (srcClass, dstClass) in enumerate(migrations.keys()): + segments[i,:] = (srcClass, dstClass) + + return segments, counts + + def _computeClassPoints(self, alignmentMd: emlib.MetaData) -> np.ndarray: + class3d = alignmentMd.getColumnValues(emlib.MDL_REF3D) + freq = collections.Counter(class3d) + return np.array(list(freq.items())) + + def _computeClassMigrationElements(self, alignmentFilenames: Iterator[str]) -> Tuple[np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray]: + segments = [] + points = [] + counts = [] + sizes = [] + + srcAlignmentFn = next(alignmentFilenames) + srcAlignmentMd = emlib.MetaData(srcAlignmentFn) + iterationPoints, iterationSizes = self._computeClassFrequencies(srcAlignmentMd) + counts.append(iterationCounts) + sizes.append(iterationSizes) + for iteration, dstAlignmentFn in enumerate(alignmentFilenames, start=1): + dstAlignmentMd = emlib.MetaData(dstAlignmentFn) + + srcAlignmentMd.intersection(dstAlignmentMd, emlib.MDL_ITEM_ID) + migrations = self._computeClassMigrations(srcAlignmentMd, dstAlignmentMd) + iterationSegments, iterationCounts = self._computeIterationClassMigrationSegments(iteration, migrations) + iterationPoints, iterationSizes = self._computeClassFrequencies(dstAlignmentMd) + + segments.append(iterationSegments) + points.append(iterationPoints) + counts.append(iterationCounts) + sizes.append(iterationSizes) + + # Save for next + srcAlignmentMd = dstAlignmentMd + + return np.concatenate(segments), np.concatenate(points), np.concatenate(counts), np.concatenate(sizes) + # ---------------------------- SHOW functions ------------------------------ def _showIterationFsc(self, e): fig, ax = plt.subplots() it = int(self.showIterationFsc) - for cls in itertools.count(): - fscFn = self._getFscFilename(it, cls) - if os.path.exists(fscFn): - fsc = self._readFsc(fscFn) - label = f'Class {cls}' - ax.plot(fsc[:,0], fsc[:,1], label=label) - else: - break + for cls, fscFn in enumerate(self._iterFscMdIterationFilenames(it), start=1): + fscMd = emlib.MetaData(fscFn) + fsc = self._readFsc(fscMd) + label = f'Class {cls}' + ax.plot(fsc[:,0], fsc[:,1], label=label) ax.set_title('Class FSC') ax.set_xlabel('Resolution (1/A)') @@ -132,14 +228,11 @@ def _showClassFsc(self, e): fig, ax = plt.subplots() cls = int(self.showClassFsc) - for it in itertools.count(): - fscFn = self._getFscFilename(it, cls) - if os.path.exists(fscFn): - fsc = self._readFsc(fscFn) - label = f'Iteration {it}' - ax.plot(fsc[:,0], fsc[:,1], label=label) - else: - break + for it, fscFn in enumerate(self._iterFscMdClassFilenames(cls), start=1): + fscMd = emlib.MetaData(fscFn) + fsc = self._readFsc(fscMd) + label = f'Iteration {it}' + ax.plot(fsc[:,0], fsc[:,1], label=label) ax.set_title('Class FSC') ax.set_xlabel('Resolution (1/A)') @@ -163,4 +256,23 @@ def _showFscCutoff(self, e): ax.set_ylabel('Resolution (A)') ax.legend() - return [fig] \ No newline at end of file + return [fig] + + def _showClassMigration(self, e): + fig, ax = plt.subplots() + + it = self._iterAlignmentMdFilenames() + segments, counts = self._computeClassMigrationSegments(it) + lines = matplotlib.collections.LineCollection(segments, array=counts) + + it = self._iterAlignmentMdFilenames() + points, sizes = self._computeClassSizePoints(it) + + ax.add_collection(lines) + sc = ax.scatter(points[:,0], points[:,1], c=sizes) + ax.set_xlabel('Iteration') + ax.set_ylabel('Class') + fig.colorbar(sc, ax=ax) + + return [fig] + \ No newline at end of file From fdb8b471231a6d645b591c50be52212315451e1a Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 28 Feb 2023 09:56:28 +0100 Subject: [PATCH 039/104] Disabled local refinement temporally --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 994e0c5a4..001279b11 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -334,7 +334,8 @@ def alignStep(self, iteration: int): args += ['--max_size', self.databaseMaximumSize] if self.useGpu: args += ['--device', 'cuda:0'] # TODO select - if iteration > 0 or self.considerInputAlignment: + #if iteration > 0 or self.considerInputAlignment: + if False: args += ['--local_shift', '--local_psi'] env = self.getCondaEnv() @@ -690,12 +691,12 @@ def _computeShiftStep(self, digital_freq: float, eps: float = 0.5) -> float: return (0.5 / digital_freq) * eps def _getIterationMaxPsi(self, iteration: int) -> float: - return float(self.initialMaxPsi) / math.pow(2.0, iteration) - #return float(self.initialMaxPsi) + #return float(self.initialMaxPsi) / math.pow(2.0, iteration) + return float(self.initialMaxPsi) def _getIterationMaxShift(self, iteration: int) -> float: - return float(self.initialMaxShift) / math.pow(2.0, iteration) - #return float(self.initialMaxShift) + #return float(self.initialMaxShift) / math.pow(2.0, iteration) + return float(self.initialMaxShift) def _createOutputClasses3D(self, volumes: SetOfVolumes): particles = self._createSetOfParticles() From 3a4e1f29bf51cae946b9fa0f56aa2efd5fc28071 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 28 Feb 2023 11:50:13 +0100 Subject: [PATCH 040/104] Added lower limit to resolution cutoff graph --- xmipp3/viewers/viewer_reconstruct_swiftres.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index 67033eb80..71e150039 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -254,6 +254,7 @@ def _showFscCutoff(self, e): ax.set_title('Resolution') ax.set_xlabel('Iteration') ax.set_ylabel('Resolution (A)') + ax.set_ylim(bottom=0) ax.legend() return [fig] From 4741425724bb81f9108d5cf8953d396c7b592a1d Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 28 Feb 2023 14:28:10 +0100 Subject: [PATCH 041/104] Added formatting to frequency axis in the FSC --- xmipp3/viewers/viewer_reconstruct_swiftres.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index 71e150039..0dc8deb2d 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -144,6 +144,12 @@ def _readFscCutoff(self, cls: int, cutoff: float) -> np.ndarray: return np.array(values) + def _computeFscTickLabels(self, ticks: np.ndarray) -> np.ndarray: + samplingRate = self._getSamplingRate() + print(ticks) + labels = list(map('1/{0:.2f}'.format, samplingRate/ticks)) + return labels + def _computeClassMigrations(self, srcMd: emlib.MetaData, dstMd: emlib.MetaData) -> dict: result = {} @@ -220,6 +226,7 @@ def _showIterationFsc(self, e): ax.set_title('Class FSC') ax.set_xlabel('Resolution (1/A)') ax.set_ylabel('FSC') + ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) ax.legend() return [fig] @@ -237,6 +244,7 @@ def _showClassFsc(self, e): ax.set_title('Class FSC') ax.set_xlabel('Resolution (1/A)') ax.set_ylabel('FSC') + ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) ax.legend() return [fig] From 8ad82954547ba00c59ccdf8855d9808da5c48635 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 28 Feb 2023 14:28:37 +0100 Subject: [PATCH 042/104] Removed print --- xmipp3/viewers/viewer_reconstruct_swiftres.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index 0dc8deb2d..1b4e824ab 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -146,7 +146,6 @@ def _readFscCutoff(self, cls: int, cutoff: float) -> np.ndarray: def _computeFscTickLabels(self, ticks: np.ndarray) -> np.ndarray: samplingRate = self._getSamplingRate() - print(ticks) labels = list(map('1/{0:.2f}'.format, samplingRate/ticks)) return labels From 0d8bcf9988ef8830782cff3af10a33d974f841db Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 28 Feb 2023 14:38:21 +0100 Subject: [PATCH 043/104] Updated iteration parameters --- .../protocol_reconstruct_swiftres.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 001279b11..29c75bc55 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -124,7 +124,7 @@ def _insertAllSteps(self): def _insertIterationSteps(self, iteration: int, prerequisites): setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, prerequisites=prerequisites) projectIds = self._insertProjectSteps(iteration, prerequisites=[setupIterationStepId]) - alignIds = self._insertAlignmentSteps(iteration, prerequisites=projectIds) + alignIds = self._insertGlobalAlignmentSteps(iteration, prerequisites=projectIds) ids = [] for cls in range(self._getClassCount()): @@ -145,7 +145,7 @@ def _insertProjectSteps(self, iteration: int, prerequisites): return [mergeGalleriesStepId] - def _insertAlignmentSteps(self, iteration: int, prerequisites): + def _insertGlobalAlignmentSteps(self, iteration: int, prerequisites): trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=prerequisites) alignStepId = self._insertFunctionStep('alignStep', iteration, prerequisites=[trainDatabaseStepId]) @@ -211,7 +211,7 @@ def setupIterationStep(self, iteration: int): args += ['--set', 'union', self._getIterationInputParticleMdFilename(iteration), 'itemId'] self._runMdUtils(args) - resolutionLimit = self._computeIterationResolution(iteration-1) + resolution = self._computeIterationResolution(iteration-1) else: # For the first iteration, simply use the input particles. @@ -220,18 +220,20 @@ def setupIterationStep(self, iteration: int): self._getIterationInputParticleMdFilename(iteration) ) - resolutionLimit = float(self.initialResolution) + resolution = float(self.initialResolution) - maxFrequency = self._getSamplingRate() / resolutionLimit + frequency = self._getSamplingRate() / resolution maxPsi = self._getIterationMaxPsi(iteration) maxShift = self._getIterationMaxShift(iteration) - shiftStep = self._computeShiftStep(maxFrequency) - angleStep = self._computeAngleStep(maxFrequency, 160) #TODO + shiftStep = self._computeShiftStep(frequency) + angleStep = self._computeAngleStep(frequency, 160) #TODO + maxResolution = max(resolution, float(self.maximumResolution)) + maxFrequency = self._getSamplingRate() / maxResolution # Write to metadata md = emlib.MetaData() id = md.addObject() - md.setValue(emlib.MDL_RESOLUTION_FREQ, resolutionLimit, id) + md.setValue(emlib.MDL_RESOLUTION_FREQ, maxResolution, id) md.setValue(emlib.MDL_RESOLUTION_FREQREAL, maxFrequency, id) md.setValue(emlib.MDL_ANGLE_PSI, maxPsi, id) md.setValue(emlib.MDL_SHIFT_X, maxShift, id) @@ -670,7 +672,7 @@ def _computeIterationResolution(self, iteration: int) -> float: res /= self._getClassCount() - return max(res, float(self.maximumResolution)) + return res def _computeAngleStep(self, maxFrequency: float, size: int) -> float: # At the alignment resolution limit, determine the From 3d6607848616356d31bb942963b499ff5b71cb74 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 1 Mar 2023 13:07:30 +0100 Subject: [PATCH 044/104] Added CTF group calls for each iteration --- .../protocol_reconstruct_swiftres.py | 55 +++++++++++++++---- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 29c75bc55..62a55420e 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -117,14 +117,15 @@ def _insertAllSteps(self): lastIds = [correctCtfStepId] for i in range(self._getIterationCount()): - lastIds = self._insertIterationSteps(i, prerequisites=lastIds) + lastIds = self._insertIterationSteps(i, False, prerequisites=lastIds) self._insertFunctionStep('createOutputStep', prerequisites=lastIds) - def _insertIterationSteps(self, iteration: int, prerequisites): - setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, prerequisites=prerequisites) + def _insertIterationSteps(self, iteration: int, local: bool, prerequisites): + setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, local, prerequisites=prerequisites) + ctfGroupStepId = self._insertFunctionStep('ctfGroupStep', iteration, prerequisites=[setupIterationStepId]) projectIds = self._insertProjectSteps(iteration, prerequisites=[setupIterationStepId]) - alignIds = self._insertGlobalAlignmentSteps(iteration, prerequisites=projectIds) + alignIds = self._insertAlignmentSteps(iteration, local, prerequisites=projectIds + [ctfGroupStepId]) ids = [] for cls in range(self._getClassCount()): @@ -145,9 +146,9 @@ def _insertProjectSteps(self, iteration: int, prerequisites): return [mergeGalleriesStepId] - def _insertGlobalAlignmentSteps(self, iteration: int, prerequisites): + def _insertAlignmentSteps(self, iteration: int, local: bool, prerequisites): trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=prerequisites) - alignStepId = self._insertFunctionStep('alignStep', iteration, prerequisites=[trainDatabaseStepId]) + alignStepId = self._insertFunctionStep('alignStep', iteration, local, prerequisites=[trainDatabaseStepId]) return [alignStepId] @@ -184,12 +185,12 @@ def correctCtfStep(self): args += ['--sampling_rate', self._getSamplingRate()] args += ['--pad', '2'] args += ['--wc', '-1.0'] - if (particles.isPhaseFlipped()): + if particles.isPhaseFlipped(): args += ['--phase_flipped'] self.runJob('xmipp_ctf_correct_wiener2d', args) - def setupIterationStep(self, iteration: int): + def setupIterationStep(self, iteration: int, local: bool): makePath(self._getIterationPath(iteration)) for cls in range(self._getClassCount()): @@ -242,6 +243,30 @@ def setupIterationStep(self, iteration: int): md.setValue(emlib.MDL_ANGLE_DIFF, angleStep, id) md.write(self._getIterationParametersFilename(iteration)) + def ctfGroupStep(self, iteration: int): + md = emlib.MetaData(self._getIterationParametersFilename(iteration)) + resolution = md.getValue(emlib.MDL_RESOLUTION_FREQ, 1) + particles = self.inputParticles.get() + oroot = self._getCtfGroupOutputRoot(iteration) + + args = [] + args += ['--ctfdat', self._getInputParticleMdFilename()] + args += ['-o', oroot] + args += ['--sampling_rate', self._getSamplingRate()] + args += ['--pad', 2] + args += ['--error', 1.0] # TODO make param + args += ['--resol', resolution] + if particles.isPhaseFlipped(): + args += ['--phase_flipped'] + + self.runJob('xmipp_ctf_group', args, numberOfMpi=1) + + # Rename files and removed unused ones + cleanPath(oroot + 'Info.xmd') + cleanPath(oroot + '_split.doc') + moveFile(oroot + '_images.sel', self._getCtfGroupMdFilename(iteration)) + moveFile(oroot + '_ctf.mrcs', self._getCtfGroupAveragesFilename(iteration)) + def projectVolumeStep(self, iteration: int, cls: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) angleStep = md.getValue(emlib.MDL_ANGLE_DIFF, 1) @@ -308,7 +333,7 @@ def trainDatabaseStep(self, iteration: int): env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_train_database', args, numberOfMpi=1, env=env) - def alignStep(self, iteration: int): + def alignStep(self, iteration: int, local: bool): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) imageSize = 160 # TODO determine maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) @@ -336,8 +361,7 @@ def alignStep(self, iteration: int): args += ['--max_size', self.databaseMaximumSize] if self.useGpu: args += ['--device', 'cuda:0'] # TODO select - #if iteration > 0 or self.considerInputAlignment: - if False: + if local: args += ['--local_shift', '--local_psi'] env = self.getCondaEnv() @@ -584,6 +608,15 @@ def _getIterationInputVolumeFilename(self, iteration: int, cls: int): def _getIterationInputParticleMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'input_particles.xmd') + def _getCtfGroupOutputRoot(self, iteration: int): + return self._getIterationPath(iteration, 'ctf_group') + + def _getCtfGroupMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'ctf_groups.xmd') + + def _getCtfGroupAveragesFilename(self, iteration: int): + return self._getIterationPath(iteration, 'ctfs.mrcs') + def _getClassGalleryMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'gallery.doc') From 0da50e38df13f95aaee01dfee1a1637a968f858a Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 2 Mar 2023 13:13:58 +0100 Subject: [PATCH 045/104] Refactored angle comparisons and added viewer --- .../protocol_reconstruct_swiftres.py | 72 +++++------ xmipp3/viewers/viewer_reconstruct_swiftres.py | 121 ++++++++++++------ 2 files changed, 118 insertions(+), 75 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 62a55420e..f06f34709 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -126,14 +126,15 @@ def _insertIterationSteps(self, iteration: int, local: bool, prerequisites): ctfGroupStepId = self._insertFunctionStep('ctfGroupStep', iteration, prerequisites=[setupIterationStepId]) projectIds = self._insertProjectSteps(iteration, prerequisites=[setupIterationStepId]) alignIds = self._insertAlignmentSteps(iteration, local, prerequisites=projectIds + [ctfGroupStepId]) - + compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, prerequisites=alignIds) + ids = [] for cls in range(self._getClassCount()): reconstructIds = self._insertReconstructSteps(iteration, cls, prerequisites=alignIds) postProcessIds = self._insertPostProcessSteps(iteration, cls, prerequisites=reconstructIds) ids += postProcessIds - return ids + return ids + [compareAnglesStepId] def _insertProjectSteps(self, iteration: int, prerequisites): # Project all volumes @@ -149,13 +150,13 @@ def _insertProjectSteps(self, iteration: int, prerequisites): def _insertAlignmentSteps(self, iteration: int, local: bool, prerequisites): trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=prerequisites) alignStepId = self._insertFunctionStep('alignStep', iteration, local, prerequisites=[trainDatabaseStepId]) + intersectInputStepId = self._insertFunctionStep('intersectInputStep', iteration, prerequisites=[alignStepId]) - return [alignStepId] + return [intersectInputStepId] def _insertReconstructSteps(self, iteration: int, cls: int, prerequisites): selectAlignmentStepId = self._insertFunctionStep('selectAlignmentStep', iteration, cls, prerequisites=prerequisites) - compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, cls, prerequisites=[selectAlignmentStepId]) - compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[compareAnglesStepId]) + compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[selectAlignmentStepId]) computeWeightsStepId = self._insertFunctionStep('computeWeightsStep', iteration, cls, prerequisites=[compareReprojectionStepId]) #filterByWeightsStepId = self._insertFunctionStep('filterByWeightsStep', iteration, cls, prerequisites=[computeWeightsStepId]) splitStepId = self._insertFunctionStep('splitStep', iteration, cls, prerequisites=[computeWeightsStepId]) @@ -368,31 +369,33 @@ def alignStep(self, iteration: int, local: bool): env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) - def selectAlignmentStep(self, iteration: int, cls: int): - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['-o', self._getReconstructionMdFilename(iteration, cls)] - args += ['--query', 'select', 'ref3d==%d' % (cls+1)] - self._runMdUtils(args) - + def intersectInputStep(self, iteration: int): args = [] args += ['-i', self._getIterationInputParticleMdFilename(iteration)] - args += ['-o', self._getInputIntersectionMdFilename(iteration, cls)] - args += ['--set', 'intersection', self._getReconstructionMdFilename(iteration, cls), 'itemId'] + args += ['-o', self._getInputIntersectionMdFilename(iteration)] + args += ['--set', 'intersection', self._getAlignmentMdFilename(iteration), 'itemId'] self._runMdUtils(args) - def compareAnglesStep(self, iteration: int, cls): + def compareAnglesStep(self, iteration: int): + oroot = self._getAngleDiffOutputRoot(iteration) + args = [] - args += ['--ang1', self._getReconstructionMdFilename(iteration, cls)] - args += ['--ang2', self._getInputIntersectionMdFilename(iteration, cls)] - args += ['--oroot', self._getReconstructionAngleDiffOutputRoot(iteration, cls)] + args += ['--ang1', self._getAlignmentMdFilename(iteration)] + args += ['--ang2', self._getInputIntersectionMdFilename(iteration)] + args += ['--oroot', oroot] args += ['--sym', self.symmetryGroup] self.runJob('xmipp_angular_distance', args, numberOfMpi=1) - #args = [] - #args += ['-i', self._getReconstructionMdFilename(iteration, cls)] - #args += ['--set', 'join', self._getReconstructionAngleDiffMdFilename(iteration, cls), 'itemId'] - #self._runMdUtils(args) + # Rename files to have xmd extension + moveFile(oroot+'_vec_diff_hist.txt', self._getAngleDiffVecDiffHistogramMdFilename(iteration)) + moveFile(oroot+'_shift_diff_hist.txt', self._getAngleDiffShiftDiffHistogramMdFilename(iteration)) + + def selectAlignmentStep(self, iteration: int, cls: int): + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['-o', self._getReconstructionMdFilename(iteration, cls)] + args += ['--query', 'select', 'ref3d==%d' % (cls+1)] + self._runMdUtils(args) def compareReprojectionStep(self, iteration: int, cls: int): args = [] @@ -519,12 +522,6 @@ def mergeAlignmentsStep(self, iteration: int): self._getAlignmentMdFilename(iteration) ) - def mergeAnglesStep(self, iteration: int): - self._mergeMetadata( - map(lambda cls : self._getReconstructionAngleDiffMdFilename(iteration, cls), range(self._getClassCount())), - self._getAngleDiffMdFilename(iteration) - ) - def createOutputStep(self): lastIteration = self._getIterationCount() - 1 @@ -635,17 +632,20 @@ def _getTrainingIndexFilename(self, iteration: int): def _getAlignmentMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'aligned.xmd') - def _getInputIntersectionMdFilename(self, iteration: int, cls: int): - return self._getClassPath(iteration, cls, 'input_intersection.xmd') + def _getInputIntersectionMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'input_intersection.xmd') - def _getReconstructionAngleDiffOutputRoot(self, iteration: int, cls: int, suffix=''): - return self._getClassPath(iteration, cls, 'angles'+suffix) - - def _getReconstructionAngleDiffMdFilename(self, iteration: int, cls: int): - return self._getReconstructionAngleDiffOutputRoot(iteration, cls, '.xmd') + def _getAngleDiffOutputRoot(self, iteration: int): + return self._getIterationPath(iteration, 'angles') def _getAngleDiffMdFilename(self, iteration: int): - return self._getIterationPath(iteration, 'angles.xmd') + return self._getAngleDiffOutputRoot(iteration) + '.xmd' + + def _getAngleDiffVecDiffHistogramMdFilename(self, iteration: int): + return self._getAngleDiffOutputRoot(iteration) + '_vec_diff_hist.xmd' + + def _getAngleDiffShiftDiffHistogramMdFilename(self, iteration: int): + return self._getAngleDiffOutputRoot(iteration) + '_shift_diff_hist.xmd' def _getReconstructionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'aligned.xmd') diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index 1b4e824ab..a60231ab2 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -24,7 +24,7 @@ # * # ************************************************************************** -from typing import Tuple, Iterator +from typing import Tuple, Iterator, Iterable import os import itertools import collections @@ -60,6 +60,10 @@ def _defineParams(self, form): form.addParam('showFscCutoff', FloatParam, label='Display FSC cutoff', default=0.143) + form.addSection(label='Angular difference') + form.addParam('showAngularDiffVecDiffHist', LabelParam, label='Display vector difference histogram') + form.addParam('showAngularDiffShiftDiffHist', LabelParam, label='Display shift difference histogram') + form.addSection(label='Classification') form.addParam('showClassMigration', LabelParam, label='Display class migration diagram', default=0.143) @@ -70,6 +74,9 @@ def _getVisualizeDict(self): 'showClassFsc': self._showClassFsc, 'showFscCutoff': self._showFscCutoff, + 'showAngularDiffVecDiffHist': self._showAngleDiffVecDiffHistogram, + 'showAngularDiffShiftDiffHist': self._showAngleDiffShiftDiffHistogram, + 'showClassMigration': self._showClassMigration } @@ -90,6 +97,12 @@ def _getAlignmentMdFilename(self, iteration: int): def _getFscFilename(self, iteration: int, cls: int): return self.protocol._getFscFilename(iteration, cls) + def _getAngleDiffVecDiffHistogramMdFilename(self, iteration: int): + return self.protocol._getAngleDiffVecDiffHistogramMdFilename(iteration) + + def _getAngleDiffShiftDiffHistogramMdFilename(self, iteration: int): + return self.protocol._getAngleDiffShiftDiffHistogramMdFilename(iteration) + def _readFsc(self, fscMd: emlib.MetaData) -> np.ndarray: values = [] for objId in fscMd: @@ -99,32 +112,33 @@ def _readFsc(self, fscMd: emlib.MetaData) -> np.ndarray: return np.array(values) - def _iterAlignmentMdFilenames(self): - for it in itertools.count(): - alignmentFn = self._getAlignmentMdFilename(it) - if os.path.exists(alignmentFn): - yield alignmentFn + def _iterFilenamesUntilNotExist(self, filenames: Iterable[str]): + for filename in filenames: + if os.path.exists(filename): + yield filename else: break def _iterFscMdIterationFilenames(self, it: int): - for cls in itertools.count(): - fscFn = self._getFscFilename(it, cls) - if os.path.exists(fscFn): - yield fscFn - - else: - break + return self._iterFilenamesUntilNotExist( + map(lambda cls : self._getFscFilename(it, cls), itertools.count()) + ) def _iterFscMdClassFilenames(self, cls: int): - for it in itertools.count(): - fscFn = self._getFscFilename(it, cls) - if os.path.exists(fscFn): - yield fscFn - - else: - break + return self._iterFilenamesUntilNotExist( + map(lambda it : self._getFscFilename(it, cls), itertools.count()) + ) + + def _iterAngleDiffVecDiffHistMdFilenames(self): + return self._iterFilenamesUntilNotExist( + map(lambda it : self._getAngleDiffVecDiffHistogramMdFilename(it), itertools.count()) + ) + + def _iterAngleDiffShiftDiffHistMdFilenames(self): + return self._iterFilenamesUntilNotExist( + map(lambda it : self._getAngleDiffShiftDiffHistogramMdFilename(it), itertools.count()) + ) def _readFscCutoff(self, cls: int, cutoff: float) -> np.ndarray: samplingRate = self._getSamplingRate() @@ -179,37 +193,36 @@ def _computeClassPoints(self, alignmentMd: emlib.MetaData) -> np.ndarray: freq = collections.Counter(class3d) return np.array(list(freq.items())) - def _computeClassMigrationElements(self, alignmentFilenames: Iterator[str]) -> Tuple[np.ndarray, - np.ndarray, - np.ndarray, - np.ndarray]: - segments = [] + def _computeClassMigrationElements(self, alignmentFilenames: Iterator[str]) -> np.ndarray: points = [] - counts = [] - sizes = [] srcAlignmentFn = next(alignmentFilenames) srcAlignmentMd = emlib.MetaData(srcAlignmentFn) - iterationPoints, iterationSizes = self._computeClassFrequencies(srcAlignmentMd) - counts.append(iterationCounts) - sizes.append(iterationSizes) for iteration, dstAlignmentFn in enumerate(alignmentFilenames, start=1): dstAlignmentMd = emlib.MetaData(dstAlignmentFn) srcAlignmentMd.intersection(dstAlignmentMd, emlib.MDL_ITEM_ID) - migrations = self._computeClassMigrations(srcAlignmentMd, dstAlignmentMd) - iterationSegments, iterationCounts = self._computeIterationClassMigrationSegments(iteration, migrations) - iterationPoints, iterationSizes = self._computeClassFrequencies(dstAlignmentMd) - - segments.append(iterationSegments) - points.append(iterationPoints) - counts.append(iterationCounts) - sizes.append(iterationSizes) + + # TODO # Save for next srcAlignmentMd = dstAlignmentMd - return np.concatenate(segments), np.concatenate(points), np.concatenate(counts), np.concatenate(sizes) + return np.concatenate(points) + + def _readAngleDiffVecHistogram(self, filename): + md = emlib.MetaData(filename) + diff = md.getColumnValues(emlib.MDL_ANGLE_DIFF) + count = md.getColumnValues(emlib.MDL_COUNT) + + return diff, count + + def _readAngleDiffShiftHistogram(self, filename): + md = emlib.MetaData(filename) + diff = md.getColumnValues(emlib.MDL_SHIFT_DIFF) + count = md.getColumnValues(emlib.MDL_COUNT) + + return diff, count # ---------------------------- SHOW functions ------------------------------ def _showIterationFsc(self, e): @@ -266,6 +279,36 @@ def _showFscCutoff(self, e): return [fig] + def _showAngleDiffVecDiffHistogram(self, e): + fig, ax = plt.subplots() + + for it, fn in enumerate(self._iterAngleDiffVecDiffHistMdFilenames()): + diff, count = self._readAngleDiffVecHistogram(fn) + label = f'Iteration {it}' + ax.plot(diff, count, label=label) + + ax.set_title('Angle difference histogram') + ax.set_xlabel('Angle difference [deg]') + ax.set_ylabel('Particle count') + ax.legend() + + return [fig] + + def _showAngleDiffShiftDiffHistogram(self, e): + fig, ax = plt.subplots() + + for it, fn in enumerate(self._iterAngleDiffShiftDiffHistMdFilenames()): + diff, count = self._readAngleDiffShiftHistogram(fn) + label = f'Iteration {it}' + ax.plot(diff, count, label=label) + + ax.set_title('Shift difference histogram') + ax.set_xlabel('Shift difference [px]') + ax.set_ylabel('Particle count') + ax.legend() + + return [fig] + def _showClassMigration(self, e): fig, ax = plt.subplots() From 92d940b4bfb803a944488f837dfa19938ae193c1 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 2 Mar 2023 13:21:22 +0100 Subject: [PATCH 046/104] Added table view to angular difference --- xmipp3/viewers/viewer_reconstruct_swiftres.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index a60231ab2..d32c37a99 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -36,7 +36,7 @@ from pyworkflow.protocol.params import LabelParam, IntParam, FloatParam from pwem import emlib -from pwem.viewers.views import DataView +from pwem.viewers.views import DataView, ObjectView from xmipp3.protocols.protocol_reconstruct_swiftres import XmippProtReconstructSwiftres @@ -61,8 +61,10 @@ def _defineParams(self, form): default=0.143) form.addSection(label='Angular difference') - form.addParam('showAngularDiffVecDiffHist', LabelParam, label='Display vector difference histogram') - form.addParam('showAngularDiffShiftDiffHist', LabelParam, label='Display shift difference histogram') + form.addParam('showAngleDiffMetadata', IntParam, label='Display iteration angular difference metadata', + default=0) + form.addParam('showAngleDiffVecDiffHist', LabelParam, label='Display vector difference histogram') + form.addParam('showAngleDiffShiftDiffHist', LabelParam, label='Display shift difference histogram') form.addSection(label='Classification') form.addParam('showClassMigration', LabelParam, label='Display class migration diagram', @@ -74,8 +76,9 @@ def _getVisualizeDict(self): 'showClassFsc': self._showClassFsc, 'showFscCutoff': self._showFscCutoff, - 'showAngularDiffVecDiffHist': self._showAngleDiffVecDiffHistogram, - 'showAngularDiffShiftDiffHist': self._showAngleDiffShiftDiffHistogram, + 'showAngleDiffMetadata': self._showAngleDiffMetadata, + 'showAngleDiffVecDiffHist': self._showAngleDiffVecDiffHistogram, + 'showAngleDiffShiftDiffHist': self._showAngleDiffShiftDiffHistogram, 'showClassMigration': self._showClassMigration } @@ -97,6 +100,9 @@ def _getAlignmentMdFilename(self, iteration: int): def _getFscFilename(self, iteration: int, cls: int): return self.protocol._getFscFilename(iteration, cls) + def _getAngleDiffMdFilename(self, iteration: int): + return self.protocol._getAngleDiffMdFilename(iteration) + def _getAngleDiffVecDiffHistogramMdFilename(self, iteration: int): return self.protocol._getAngleDiffVecDiffHistogramMdFilename(iteration) @@ -279,6 +285,11 @@ def _showFscCutoff(self, e): return [fig] + def _showAngleDiffMetadata(self, e): + mdFilename = self._getAngleDiffMdFilename(int(self.showAngleDiffMetadata)) + v = DataView(mdFilename) + return [v] + def _showAngleDiffVecDiffHistogram(self, e): fig, ax = plt.subplots() From 3fbad820b5e26a3b8ec5388ae35fd5621d789c5e Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 3 Mar 2023 09:39:17 +0100 Subject: [PATCH 047/104] Implemented alignment consensus --- .../protocol_reconstruct_swiftres.py | 140 ++++++++++++++---- 1 file changed, 111 insertions(+), 29 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index f06f34709..4c4a42d69 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -73,6 +73,7 @@ def _defineParams(self, form: Form): form.addSection(label='Global refinement') form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) + form.addParam('numberOfAlignmentRepetitions', IntParam, label='Number of repetitions', default=2) form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0, help='Image comparison resolution limit at the first iteration of the refinement') form.addParam('maximumResolution', FloatParam, label="Maximum resolution (A)", default=8.0, @@ -137,20 +138,30 @@ def _insertIterationSteps(self, iteration: int, local: bool, prerequisites): return ids + [compareAnglesStepId] def _insertProjectSteps(self, iteration: int, prerequisites): - # Project all volumes - projectStepIds = [] - for cls in range(self._getClassCount()): - projectStepIds.append(self._insertFunctionStep('projectVolumeStep', iteration, cls, prerequisites=prerequisites)) - - # Merge galleries - mergeGalleriesStepId = self._insertFunctionStep('mergeGalleriesStep', iteration, prerequisites=projectStepIds) + mergeStepIds = [] + + for repetition in range(self._getAlignmentRepetitionCount()): + projectStepIds = [] + + # Project all classes + for cls in range(self._getClassCount()): + projectStepIds.append(self._insertFunctionStep('projectVolumeStep', iteration, cls, repetition, prerequisites=prerequisites)) + + # Merge galleries of classes + mergeStepIds.append(self._insertFunctionStep('mergeGalleriesStep', iteration, repetition, prerequisites=projectStepIds)) - return [mergeGalleriesStepId] + return mergeStepIds def _insertAlignmentSteps(self, iteration: int, local: bool, prerequisites): - trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=prerequisites) - alignStepId = self._insertFunctionStep('alignStep', iteration, local, prerequisites=[trainDatabaseStepId]) - intersectInputStepId = self._insertFunctionStep('intersectInputStep', iteration, prerequisites=[alignStepId]) + ensembleTrainingSetStepId = self._insertFunctionStep('ensembleTrainingSetStep', iteration, prerequisites=prerequisites) + trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=[ensembleTrainingSetStepId]) + + alignStepIds = [] + for repetition in range(self._getAlignmentRepetitionCount()): + alignStepIds.append(self._insertFunctionStep('alignStep', iteration, repetition, local, prerequisites=[trainDatabaseStepId])) + + alignmentConsensusStepId = self._insertFunctionStep('alignmentConsensusStep', iteration, prerequisites=alignStepIds) + intersectInputStepId = self._insertFunctionStep('intersectInputStep', iteration, prerequisites=[alignmentConsensusStepId]) return [intersectInputStepId] @@ -268,14 +279,14 @@ def ctfGroupStep(self, iteration: int): moveFile(oroot + '_images.sel', self._getCtfGroupMdFilename(iteration)) moveFile(oroot + '_ctf.mrcs', self._getCtfGroupAveragesFilename(iteration)) - def projectVolumeStep(self, iteration: int, cls: int): + def projectVolumeStep(self, iteration: int, cls: int, repetition: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) angleStep = md.getValue(emlib.MDL_ANGLE_DIFF, 1) perturb = math.sin(math.radians(angleStep)) / 4 args = [] args += ['-i', self._getIterationInputVolumeFilename(iteration, cls)] - args += ['-o', self._getClassGalleryStackFilename(iteration, cls)] + args += ['-o', self._getClassGalleryStackFilename(iteration, cls, repetition)] args += ['--sampling_rate', angleStep] args += ['--perturb', perturb] args += ['--sym', self.symmetryGroup] @@ -288,22 +299,28 @@ def projectVolumeStep(self, iteration: int, cls: int): self.runJob('xmipp_angular_project_library', args) args = [] - args += ['-i', self._getClassGalleryMdFilename(iteration, cls)] + args += ['-i', self._getClassGalleryMdFilename(iteration, cls, repetition)] args += ['--fill', 'ref3d', 'constant', cls+1] self._runMdUtils(args) - def mergeGalleriesStep(self, iteration): + def mergeGalleriesStep(self, iteration: int, repetition: int): self._mergeMetadata( - map(lambda cls : self._getClassGalleryMdFilename(iteration, cls), range(self._getClassCount())), - self._getGalleryMdFilename(iteration) + map(lambda cls : self._getClassGalleryMdFilename(iteration, cls, repetition), range(self._getClassCount())), + self._getGalleryMdFilename(iteration, repetition) ) # Reindex args = [] - args += ['-i', self._getGalleryMdFilename(iteration)] + args += ['-i', self._getGalleryMdFilename(iteration, repetition)] args += ['--fill', 'ref', 'lineal', 1, 1] self._runMdUtils(args) - + + def ensembleTrainingSetStep(self, iteration: int): + self._mergeMetadata( + map(lambda i : self._getGalleryMdFilename(iteration, i), range(self._getAlignmentRepetitionCount())), + self._getTrainingMdFilename(iteration) + ) + def trainDatabaseStep(self, iteration: int): trainingSize = int(self.databaseTrainingSetSize) recipe = self.databaseRecipe @@ -316,7 +333,7 @@ def trainDatabaseStep(self, iteration: int): maxShift = maxShiftPx / imageSize args = [] - args += ['-i', self._getGalleryMdFilename(iteration)] + args += ['-i', self._getTrainingMdFilename(iteration)] args += ['-o', self._getTrainingIndexFilename(iteration)] args += ['--recipe', recipe] #args += ['--weights', self._getWeightsFilename(iteration)] @@ -334,7 +351,7 @@ def trainDatabaseStep(self, iteration: int): env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_train_database', args, numberOfMpi=1, env=env) - def alignStep(self, iteration: int, local: bool): + def alignStep(self, iteration: int, repetition: int, local: bool): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) imageSize = 160 # TODO determine maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) @@ -347,8 +364,8 @@ def alignStep(self, iteration: int, local: bool): # Perform the alignment args = [] args += ['-i', self._getIterationInputParticleMdFilename(iteration)] - args += ['-o', self._getAlignmentMdFilename(iteration)] - args += ['-r', self._getGalleryMdFilename(iteration)] + args += ['-o', self._getAlignmentRepetitionMdFilename(iteration, repetition)] + args += ['-r', self._getGalleryMdFilename(iteration, repetition)] args += ['--index', self._getTrainingIndexFilename(iteration)] #args += ['--weights', self._getWeightsFilename(iteration)] args += ['--max_shift', maxShift] @@ -369,6 +386,50 @@ def alignStep(self, iteration: int, local: bool): env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) + def alignmentConsensusStep(self, iteration: int): + repetitionCount = self._getAlignmentRepetitionCount() + + if repetitionCount == 1: + copyFile( + self._getAlignmentRepetitionMdFilename(iteration, 0), + self._getAlignmentMdFilename(iteration), + ) + elif repetitionCount == 2: + md = emlib.MetaData(self._getIterationParametersFilename(iteration)) + angleStep = md.getValue(emlib.MDL_ANGLE_DIFF, 1) + + # Ensure that both alignments have the same particles + args = [] + args += ['-i', self._getAlignmentRepetitionMdFilename(iteration, 0)] + args += ['--set', 'intersection', self._getAlignmentRepetitionMdFilename(iteration, 1), 'itemId'] + self._runMdUtils(args) + + args = [] + args += ['-i', self._getAlignmentRepetitionMdFilename(iteration, 1)] + args += ['--set', 'intersection', self._getAlignmentRepetitionMdFilename(iteration, 0), 'itemId'] + self._runMdUtils(args) + + # Compute the angular distance + oroot = self._getAlignmentConsensusOutputRoot(iteration) + args = [] + args += ['--ang1', self._getAlignmentRepetitionMdFilename(iteration, 0)] + args += ['--ang2', self._getAlignmentRepetitionMdFilename(iteration, 1)] + args += ['--oroot', oroot] + args += ['--sym', self.symmetryGroup] + self.runJob('xmipp_angular_distance', args, numberOfMpi=1) + moveFile(oroot+'_vec_diff_hist.txt', self._getAlignmentConsensusVecDiffHistogramMdFilename(iteration)) + moveFile(oroot+'_shift_diff_hist.txt', self._getAlignmentConsensusShiftDiffHistogramMdFilename(iteration)) + + # Select particles that are less than a step apart + args = [] + args += ['-i', self._getAlignmentConsensusMdFilename(iteration)] + args += ['-o', self._getAlignmentMdFilename(iteration)] + args += ['--query', 'select', f'angleDiff <= {angleStep}'] + self._runMdUtils(args) + + else: + raise NotImplementedError('No angular consensus has been implemented for more than 2 alignments') + def intersectInputStep(self, iteration: int): args = [] args += ['-i', self._getIterationInputParticleMdFilename(iteration)] @@ -564,6 +625,9 @@ def createOutputStep(self): #--------------------------- UTILS functions -------------------------------------------- def _getIterationCount(self) -> int: return int(self.numberOfIterations) + + def _getAlignmentRepetitionCount(self) -> int: + return int(self.numberOfAlignmentRepetitions) def _getClassCount(self) -> int: return len(self.inputVolumes) @@ -614,24 +678,42 @@ def _getCtfGroupMdFilename(self, iteration: int): def _getCtfGroupAveragesFilename(self, iteration: int): return self._getIterationPath(iteration, 'ctfs.mrcs') - def _getClassGalleryMdFilename(self, iteration: int, cls: int): - return self._getClassPath(iteration, cls, 'gallery.doc') + def _getClassGalleryMdFilename(self, iteration: int, cls: int, repetition: int): + return self._getClassPath(iteration, cls, 'gallery%05d.doc' % repetition) - def _getClassGalleryStackFilename(self, iteration: int, cls: int): - return self._getClassPath(iteration, cls, 'gallery.mrcs') + def _getClassGalleryStackFilename(self, iteration: int, cls: int, repetition: int): + return self._getClassPath(iteration, cls, 'gallery%05d.mrcs' % repetition) - def _getGalleryMdFilename(self, iteration: int): - return self._getIterationPath(iteration, 'gallery.xmd') + def _getGalleryMdFilename(self, iteration: int, repetition: int): + return self._getIterationPath(iteration, 'gallery%05d.xmd' % repetition) + + def _getTrainingMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'training.xmd') def _getWeightsFilename(self, iteration: int): return self._getIterationPath(iteration, 'weights.mrc') def _getTrainingIndexFilename(self, iteration: int): return self._getIterationPath(iteration, 'database.idx') + + def _getAlignmentRepetitionMdFilename(self, iteration: int, repetition: int): + return self._getIterationPath(iteration, 'aligned%05d.xmd' % repetition) def _getAlignmentMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'aligned.xmd') + def _getAlignmentConsensusOutputRoot(self, iteration: int): + return self._getIterationPath(iteration, 'alignment_consensus') + + def _getAlignmentConsensusMdFilename(self, iteration: int): + return self._getAlignmentConsensusOutputRoot(iteration) + '.xmd' + + def _getAlignmentConsensusVecDiffHistogramMdFilename(self, iteration: int): + return self._getAlignmentConsensusOutputRoot(iteration) + '_vec_diff_hist.xmd' + + def _getAlignmentConsensusShiftDiffHistogramMdFilename(self, iteration: int): + return self._getAlignmentConsensusOutputRoot(iteration) + '_shift_diff_hist.xmd' + def _getInputIntersectionMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'input_intersection.xmd') From a80265f7455ea16cee252c08a968ebf860c691df Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 7 Mar 2023 08:54:54 +0100 Subject: [PATCH 048/104] Updated angular consensus --- .../protocol_reconstruct_swiftres.py | 100 +++++++++++------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 4c4a42d69..4ecacee26 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -35,7 +35,7 @@ import xmipp3 from xmipp3.convert import writeSetOfParticles, readSetOfParticles -from typing import Iterable +from typing import Iterable, Sequence, Optional import math class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): @@ -387,38 +387,31 @@ def alignStep(self, iteration: int, repetition: int, local: bool): self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) def alignmentConsensusStep(self, iteration: int): - repetitionCount = self._getAlignmentRepetitionCount() - - if repetitionCount == 1: + alignmentRepetitionMdFilenames = list(map( + lambda i : self._getAlignmentRepetitionMdFilename(iteration, i)), + range(self._getAlignmentRepetitionCount()) + )) + + # Ensure that both alignments have the same particles + self._keepCommonMetadataElements( + alignmentRepetitionMdFilenames + ) + + if len(alignmentRepetitionMdFilenames) == 1: copyFile( self._getAlignmentRepetitionMdFilename(iteration, 0), self._getAlignmentMdFilename(iteration), ) - elif repetitionCount == 2: + elif len(alignmentRepetitionMdFilenames) == 2: md = emlib.MetaData(self._getIterationParametersFilename(iteration)) angleStep = md.getValue(emlib.MDL_ANGLE_DIFF, 1) - # Ensure that both alignments have the same particles - args = [] - args += ['-i', self._getAlignmentRepetitionMdFilename(iteration, 0)] - args += ['--set', 'intersection', self._getAlignmentRepetitionMdFilename(iteration, 1), 'itemId'] - self._runMdUtils(args) - - args = [] - args += ['-i', self._getAlignmentRepetitionMdFilename(iteration, 1)] - args += ['--set', 'intersection', self._getAlignmentRepetitionMdFilename(iteration, 0), 'itemId'] - self._runMdUtils(args) - - # Compute the angular distance - oroot = self._getAlignmentConsensusOutputRoot(iteration) - args = [] - args += ['--ang1', self._getAlignmentRepetitionMdFilename(iteration, 0)] - args += ['--ang2', self._getAlignmentRepetitionMdFilename(iteration, 1)] - args += ['--oroot', oroot] - args += ['--sym', self.symmetryGroup] - self.runJob('xmipp_angular_distance', args, numberOfMpi=1) - moveFile(oroot+'_vec_diff_hist.txt', self._getAlignmentConsensusVecDiffHistogramMdFilename(iteration)) - moveFile(oroot+'_shift_diff_hist.txt', self._getAlignmentConsensusShiftDiffHistogramMdFilename(iteration)) + self._runAngularDistance( + alignmentRepetitionMdFilenames[0], + alignmentRepetitionMdFilenames[1], + self._getAlignmentConsensusOutputRoot(iteration), + extrargs=['--compute_average_angle', '--compute_average_shift'] + ) # Select particles that are less than a step apart args = [] @@ -438,18 +431,11 @@ def intersectInputStep(self, iteration: int): self._runMdUtils(args) def compareAnglesStep(self, iteration: int): - oroot = self._getAngleDiffOutputRoot(iteration) - - args = [] - args += ['--ang1', self._getAlignmentMdFilename(iteration)] - args += ['--ang2', self._getInputIntersectionMdFilename(iteration)] - args += ['--oroot', oroot] - args += ['--sym', self.symmetryGroup] - self.runJob('xmipp_angular_distance', args, numberOfMpi=1) - - # Rename files to have xmd extension - moveFile(oroot+'_vec_diff_hist.txt', self._getAngleDiffVecDiffHistogramMdFilename(iteration)) - moveFile(oroot+'_shift_diff_hist.txt', self._getAngleDiffShiftDiffHistogramMdFilename(iteration)) + self._runAngularDistance( + ang1=self._getAlignmentMdFilename(iteration), + ang2=self._getInputIntersectionMdFilename(iteration), + oroot=self._getAngleDiffOutputRoot(iteration) + ) def selectAlignmentStep(self, iteration: int, cls: int): args = [] @@ -908,6 +894,42 @@ def _mergeMetadata(self, src: Iterable[str], dst: str): args += ['-i', dst] args += ['--set', 'union_all', s] self._runMdUtils(args) - + + def _keepCommonMetadataElements(self, + mds: Sequence[str], + label: str = 'itemId' ): + # Intersect the first md with the rest of mds + commonMd = mds[0] # Use first item to keep track of common items + for md in mds[1:]: + args = [] + args += ['-i', commonMd] + args += ['--set', 'intersection', md, label] + self._runMdUtils(args) + + # Intersect all of the mds with the first one + for md in mds[1:]: + args = [] + args += ['-i', md] + args += ['--set', 'intersection', commonMd, label] + self._runMdUtils(args) + def _runMdUtils(self, args): self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) + + def _runAngularDistance(self, + ang1: str, + ang2: str, + oroot: str, + sym: Optional[str] = None, + extrargs = []): + args = [] + args += ['--ang1', ang1] + args += ['--ang2', ang2] + args += ['--oroot', oroot] + args += ['--sym', sym or self.symmetryGroup] + args += extrargs + self.runJob('xmipp_angular_distance', args, numberOfMpi=1) + + # Rename files to have xmd extension + moveFile(oroot+'_vec_diff_hist.txt', oroot+'_vec_diff_hist.xmd') + moveFile(oroot+'_shift_diff_hist.txt', oroot+'_shift_diff_hist.xmd') \ No newline at end of file From 7b978090f39cc0a34c93b1c6f3f057aa4ec1efde Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 9 Mar 2023 11:27:46 +0100 Subject: [PATCH 049/104] Tested angular averaging --- .../protocol_reconstruct_swiftres.py | 144 +++++++++++++----- 1 file changed, 106 insertions(+), 38 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 4ecacee26..d1d8544da 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -22,6 +22,7 @@ from pwem.protocols import ProtRefine3D from pwem.objects import Volume, FSC, SetOfVolumes, Class3D +from pwem.convert import transformations from pwem import emlib from pyworkflow.protocol.params import (Form, PointerParam, @@ -38,6 +39,10 @@ from typing import Iterable, Sequence, Optional import math +import numpy as np +from scipy import stats +import itertools + class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): _label = 'swiftres' _conda_env = 'xmipp_torch' @@ -387,41 +392,28 @@ def alignStep(self, iteration: int, repetition: int, local: bool): self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) def alignmentConsensusStep(self, iteration: int): - alignmentRepetitionMdFilenames = list(map( - lambda i : self._getAlignmentRepetitionMdFilename(iteration, i)), + paramMd = emlib.MetaData(self._getIterationParametersFilename(iteration)) + angleStep = paramMd.getValue(emlib.MDL_ANGLE_DIFF, 1) + shiftStep = paramMd.getValue(emlib.MDL_SHIFT_DIFF, 1) + + alignmentRepetitionMds = list(map( + lambda i : emlib.MetaData(self._getAlignmentRepetitionMdFilename(iteration, i)), range(self._getAlignmentRepetitionCount()) )) # Ensure that both alignments have the same particles self._keepCommonMetadataElements( - alignmentRepetitionMdFilenames + alignmentRepetitionMds, + emlib.MDL_ITEM_ID ) - if len(alignmentRepetitionMdFilenames) == 1: - copyFile( - self._getAlignmentRepetitionMdFilename(iteration, 0), - self._getAlignmentMdFilename(iteration), - ) - elif len(alignmentRepetitionMdFilenames) == 2: - md = emlib.MetaData(self._getIterationParametersFilename(iteration)) - angleStep = md.getValue(emlib.MDL_ANGLE_DIFF, 1) - - self._runAngularDistance( - alignmentRepetitionMdFilenames[0], - alignmentRepetitionMdFilenames[1], - self._getAlignmentConsensusOutputRoot(iteration), - extrargs=['--compute_average_angle', '--compute_average_shift'] - ) - - # Select particles that are less than a step apart - args = [] - args += ['-i', self._getAlignmentConsensusMdFilename(iteration)] - args += ['-o', self._getAlignmentMdFilename(iteration)] - args += ['--query', 'select', f'angleDiff <= {angleStep}'] - self._runMdUtils(args) - - else: - raise NotImplementedError('No angular consensus has been implemented for more than 2 alignments') + consensusMd = self._computeAlignmentConsensus( + alignmentRepetitionMds, + angleStep, + shiftStep + ) + + consensusMd.write(self._getAlignmentMdFilename(iteration)) def intersectInputStep(self, iteration: int): args = [] @@ -751,6 +743,88 @@ def _getOutputFscFilename(self, cls: int): def _getTrainingScratchFilename(self): return self._getTmpPath('scratch.bin') + def _averageQuaternions(self, quats: np.ndarray) -> np.ndarray: + s = np.matmul(quats.T, quats) + s /= len(quats) + eigenValues, eigenVectors = np.linalg.eig(s) + return np.real(eigenVectors[:,np.argmax(eigenValues)]) + + def _computeAlignmentConsensus( self, + mds: Sequence[emlib.MetaData], + maxAngleDiff: float, + maxShiftDiff: float ) -> emlib.MetaData: + if len(mds) > 1: + resultMd = emlib.MetaData() + + maxAngleDiff = np.deg2rad(maxAngleDiff) + + quaternions = np.empty((len(mds), 4)) + shifts = np.empty((len(mds), 2)) + ref3ds = np.empty(len(mds), dtype=int) + row = emlib.metadata.Row() + + for objIds in zip(*mds): + # Obtain the alignment data + for i, (objId, md) in enumerate(zip(objIds, mds)): + quaternions[i,:] = transformations.quaternion_from_euler( + -np.deg2rad(md.getValue(emlib.MDL_ANGLE_ROT, objId)), + -np.deg2rad(md.getValue(emlib.MDL_ANGLE_TILT, objId)), + -np.deg2rad(md.getValue(emlib.MDL_ANGLE_PSI, objId)), + axes='szyz' + ) + shifts[i,:] = ( md.getValue(emlib.MDL_SHIFT_X, objId), + md.getValue(emlib.MDL_SHIFT_Y, objId) ) + ref3ds[i] = md.getValue(emlib.MDL_REF3D, objId) + + # Perform a consensus + quaternion, _ = transformations.mean_quaternion(transformations.weighted_tensor(quaternions)) + shift = np.mean(shifts, axis=0) + ref3d = int(stats.mode(ref3ds).mode) + + # Check if more than a half agree + angleDiff = np.array(list(map(lambda q : transformations.quaternion_distance(quaternion, q), quaternions))) + #if np.count_nonzero(angleDiff <= maxAngleDiff) <= len(angleDiff) / 2: + # continue + + shiftDiff = np.linalg.norm(shifts-shift, axis=1) + if np.count_nonzero(shiftDiff <= maxShiftDiff) <= len(shiftDiff) / 2: + continue + + if np.count_nonzero(ref3ds == ref3d) <= len(ref3ds) / 2: + continue + + # All checks succeeded. Elaborate output + row.readFromMd(mds[0], objIds[0]) + + rot, tilt, psi = transformations.euler_from_quaternion(quaternion, axes='szyz') + row.setValue(emlib.MDL_ANGLE_ROT, -np.rad2deg(rot)) + row.setValue(emlib.MDL_ANGLE_TILT, -np.rad2deg(tilt)) + row.setValue(emlib.MDL_ANGLE_PSI, -np.rad2deg(psi)) + row.setValue(emlib.MDL_SHIFT_X, shift[0]) + row.setValue(emlib.MDL_SHIFT_Y, shift[1]) + row.setValue(emlib.MDL_ANGLE_DIFF, np.mean(angleDiff)) + row.setValue(emlib.MDL_SHIFT_DIFF, np.mean(shiftDiff)) + row.setValue(emlib.MDL_REF3D, ref3d) + + assert(len(mds) > 1) + row.setValue(emlib.MDL_ANGLE_PSI+1, mds[0].getValue(emlib.MDL_ANGLE_PSI, objIds[0])) + row.setValue(emlib.MDL_ANGLE_PSI+2, mds[1].getValue(emlib.MDL_ANGLE_PSI, objIds[1])) + row.setValue(emlib.MDL_ANGLE_ROT+1, mds[0].getValue(emlib.MDL_ANGLE_ROT, objIds[0])) + row.setValue(emlib.MDL_ANGLE_ROT+2, mds[1].getValue(emlib.MDL_ANGLE_ROT, objIds[1])) + row.setValue(emlib.MDL_ANGLE_TILT+1, mds[0].getValue(emlib.MDL_ANGLE_TILT, objIds[0])) + row.setValue(emlib.MDL_ANGLE_TILT+2, mds[1].getValue(emlib.MDL_ANGLE_TILT, objIds[1])) + row.setValue(emlib.MDL_SHIFT_X+1, mds[0].getValue(emlib.MDL_SHIFT_X, objIds[0])) + row.setValue(emlib.MDL_SHIFT_X+2, mds[1].getValue(emlib.MDL_SHIFT_X, objIds[1])) + row.setValue(emlib.MDL_SHIFT_Y+1, mds[0].getValue(emlib.MDL_SHIFT_Y, objIds[0])) + row.setValue(emlib.MDL_SHIFT_Y+2, mds[1].getValue(emlib.MDL_SHIFT_Y, objIds[1])) + + row.writeToMd(resultMd, resultMd.addObject()) + + return resultMd + + elif len(mds) == 1: + return mds[0] + def _computeResolution(self, mdFsc, Ts, threshold): resolution = 2 * Ts @@ -896,22 +970,16 @@ def _mergeMetadata(self, src: Iterable[str], dst: str): self._runMdUtils(args) def _keepCommonMetadataElements(self, - mds: Sequence[str], - label: str = 'itemId' ): + mds: Sequence[emlib.MetaData], + label: int = emlib.MDL_ITEM_ID ): # Intersect the first md with the rest of mds commonMd = mds[0] # Use first item to keep track of common items for md in mds[1:]: - args = [] - args += ['-i', commonMd] - args += ['--set', 'intersection', md, label] - self._runMdUtils(args) + commonMd.intersection(md, label) # Intersect all of the mds with the first one for md in mds[1:]: - args = [] - args += ['-i', md] - args += ['--set', 'intersection', commonMd, label] - self._runMdUtils(args) + md.intersection(commonMd, label) def _runMdUtils(self, args): self.runJob('xmipp_metadata_utilities', args, numberOfMpi=1) From d2a50145e64b812d61ff531c0ccabea4189931c5 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 10 Mar 2023 08:23:05 +0100 Subject: [PATCH 050/104] Updated angular consensus to use angular_distance --- .../protocol_reconstruct_swiftres.py | 74 ++++++++++++++++--- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index d1d8544da..dd7f6bbcc 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -42,6 +42,7 @@ import numpy as np from scipy import stats import itertools +import collections class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): _label = 'swiftres' @@ -129,9 +130,10 @@ def _insertAllSteps(self): def _insertIterationSteps(self, iteration: int, local: bool, prerequisites): setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, local, prerequisites=prerequisites) - ctfGroupStepId = self._insertFunctionStep('ctfGroupStep', iteration, prerequisites=[setupIterationStepId]) + #ctfGroupStepId = self._insertFunctionStep('ctfGroupStep', iteration, prerequisites=[setupIterationStepId]) projectIds = self._insertProjectSteps(iteration, prerequisites=[setupIterationStepId]) - alignIds = self._insertAlignmentSteps(iteration, local, prerequisites=projectIds + [ctfGroupStepId]) + #alignIds = self._insertAlignmentSteps(iteration, local, prerequisites=projectIds + [ctfGroupStepId]) + alignIds = self._insertAlignmentSteps(iteration, local, prerequisites=projectIds) compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, prerequisites=alignIds) ids = [] @@ -172,10 +174,11 @@ def _insertAlignmentSteps(self, iteration: int, local: bool, prerequisites): def _insertReconstructSteps(self, iteration: int, cls: int, prerequisites): selectAlignmentStepId = self._insertFunctionStep('selectAlignmentStep', iteration, cls, prerequisites=prerequisites) - compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[selectAlignmentStepId]) - computeWeightsStepId = self._insertFunctionStep('computeWeightsStep', iteration, cls, prerequisites=[compareReprojectionStepId]) + #compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[selectAlignmentStepId]) + #computeWeightsStepId = self._insertFunctionStep('computeWeightsStep', iteration, cls, prerequisites=[compareReprojectionStepId]) #filterByWeightsStepId = self._insertFunctionStep('filterByWeightsStep', iteration, cls, prerequisites=[computeWeightsStepId]) - splitStepId = self._insertFunctionStep('splitStep', iteration, cls, prerequisites=[computeWeightsStepId]) + splitStepId = self._insertFunctionStep('splitStep', iteration, cls, prerequisites=[selectAlignmentStepId]) + #splitStepId = self._insertFunctionStep('splitStep', iteration, cls, prerequisites=[computeWeightsStepId]) reconstructStepId1 = self._insertFunctionStep('reconstructStep', iteration, cls, 1, prerequisites=[splitStepId]) reconstructStepId2 = self._insertFunctionStep('reconstructStep', iteration, cls, 2, prerequisites=[splitStepId]) computeFscStepId = self._insertFunctionStep('computeFscStep', iteration, cls, prerequisites=[reconstructStepId1, reconstructStepId2]) @@ -396,17 +399,22 @@ def alignmentConsensusStep(self, iteration: int): angleStep = paramMd.getValue(emlib.MDL_ANGLE_DIFF, 1) shiftStep = paramMd.getValue(emlib.MDL_SHIFT_DIFF, 1) - alignmentRepetitionMds = list(map( - lambda i : emlib.MetaData(self._getAlignmentRepetitionMdFilename(iteration, i)), - range(self._getAlignmentRepetitionCount()) - )) - # Ensure that both alignments have the same particles + alignmentRepetitionMds = [] + for i in range(self._getAlignmentRepetitionCount()): + filename = self._getAlignmentRepetitionMdFilename(iteration, i) + md = emlib.MetaData(filename) + alignmentRepetitionMds.append(md) + self._keepCommonMetadataElements( alignmentRepetitionMds, emlib.MDL_ITEM_ID ) + for i, md in enumerate(alignmentRepetitionMds): + filename = self._getAlignmentRepetitionMdFilename(iteration, i) + md.write(filename) + """ consensusMd = self._computeAlignmentConsensus( alignmentRepetitionMds, angleStep, @@ -414,6 +422,46 @@ def alignmentConsensusStep(self, iteration: int): ) consensusMd.write(self._getAlignmentMdFilename(iteration)) + """ + if self._getAlignmentRepetitionCount() == 1: + copyFile( + self._getAlignmentRepetitionMdFilename(iteration, 0), + self._getAlignmentMdFilename(iteration) + ) + + elif self._getAlignmentRepetitionCount() == 2: + self._runAngularDistance( + self._getAlignmentRepetitionMdFilename(iteration, 0), + self._getAlignmentRepetitionMdFilename(iteration, 1), + self._getIterationPath(iteration, 'aligned'), + extrargs=['--compute_average_angle', '--compute_average_shift'] + ) + + # Drop particles far apart + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--query', 'select', 'angleDiff < %f' % angleStep] + self._runMdUtils(args) + + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--query', 'select', 'shiftDiff < %f' % shiftStep] + self._runMdUtils(args) + + # Add the missing columns + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--set', 'join', self._getInputParticleMdFilename(), 'itemId'] + self._runMdUtils(args) + + # FIXME + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--fill', 'ref3d', 'constant', 1] + + self._runMdUtils(args) + else: + raise NotImplementedError('Alignment consensus only implemented for N=2') def intersectInputStep(self, iteration: int): args = [] @@ -421,6 +469,12 @@ def intersectInputStep(self, iteration: int): args += ['-o', self._getInputIntersectionMdFilename(iteration)] args += ['--set', 'intersection', self._getAlignmentMdFilename(iteration), 'itemId'] self._runMdUtils(args) + + # Add the missing columns + args = [] + args += ['-i', self._getAlignmentMdFilename(iteration)] + args += ['--set', 'intersection', self._getAlignmentMdFilename(iteration), 'itemId'] + self._runMdUtils(args) def compareAnglesStep(self, iteration: int): self._runAngularDistance( From 0439d3f01d6b287137a7084735b5859c1cdfd9a5 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 10 Mar 2023 10:15:10 +0100 Subject: [PATCH 051/104] Added image size getter --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index dd7f6bbcc..3c2c589fa 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -243,11 +243,12 @@ def setupIterationStep(self, iteration: int, local: bool): resolution = float(self.initialResolution) + imageSize = self._getImageSize() frequency = self._getSamplingRate() / resolution maxPsi = self._getIterationMaxPsi(iteration) maxShift = self._getIterationMaxShift(iteration) shiftStep = self._computeShiftStep(frequency) - angleStep = self._computeAngleStep(frequency, 160) #TODO + angleStep = self._computeAngleStep(frequency, imageSize) maxResolution = max(resolution, float(self.maximumResolution)) maxFrequency = self._getSamplingRate() / maxResolution @@ -334,7 +335,7 @@ def trainDatabaseStep(self, iteration: int): recipe = self.databaseRecipe md = emlib.MetaData(self._getIterationParametersFilename(iteration)) - imageSize = 160 # TODO determine + imageSize = self._getImageSize() maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) maxShiftPx = md.getValue(emlib.MDL_SHIFT_X, 1) @@ -361,7 +362,7 @@ def trainDatabaseStep(self, iteration: int): def alignStep(self, iteration: int, repetition: int, local: bool): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) - imageSize = 160 # TODO determine + imageSize = self._getImageSize() maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) maxShiftPx = md.getValue(emlib.MDL_SHIFT_X, 1) @@ -670,6 +671,9 @@ def _getReconstructPercentile(self) -> float: def _getSamplingRate(self) -> float: return float(self.inputParticles.get().getSamplingRate()) + def _getImageSize(self) -> int: + return int(self.inputParticles.get().getXDim()) + def _getIterationPath(self, iteration: int, *paths): return self._getExtraPath('iteration_%04d' % iteration, *paths) From d6c614ecf173885387a5161f5a2b8812f726094e Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 10 Mar 2023 11:17:18 +0100 Subject: [PATCH 052/104] Added threshold to the viewer --- xmipp3/viewers/viewer_reconstruct_swiftres.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index d32c37a99..c2e9f946b 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -240,6 +240,8 @@ def _showIterationFsc(self, e): fsc = self._readFsc(fscMd) label = f'Class {cls}' ax.plot(fsc[:,0], fsc[:,1], label=label) + ax.axhline(0.5, color='black', linestyle='--') + ax.axhline(0.143, color='black', linestyle='--') ax.set_title('Class FSC') ax.set_xlabel('Resolution (1/A)') @@ -258,6 +260,8 @@ def _showClassFsc(self, e): fsc = self._readFsc(fscMd) label = f'Iteration {it}' ax.plot(fsc[:,0], fsc[:,1], label=label) + ax.axhline(0.5, color='black', linestyle='--') + ax.axhline(0.143, color='black', linestyle='--') ax.set_title('Class FSC') ax.set_xlabel('Resolution (1/A)') From 4822252bb7f67a5767cd053ba3bc984b0fc7c57d Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 10 Mar 2023 15:57:26 +0100 Subject: [PATCH 053/104] Continued working on alignment consensus --- .../protocol_reconstruct_swiftres.py | 71 +++++++------------ 1 file changed, 24 insertions(+), 47 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 3c2c589fa..04e9ccb75 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -415,54 +415,31 @@ def alignmentConsensusStep(self, iteration: int): for i, md in enumerate(alignmentRepetitionMds): filename = self._getAlignmentRepetitionMdFilename(iteration, i) md.write(filename) - """ - consensusMd = self._computeAlignmentConsensus( - alignmentRepetitionMds, - angleStep, - shiftStep - ) - - consensusMd.write(self._getAlignmentMdFilename(iteration)) - """ - if self._getAlignmentRepetitionCount() == 1: - copyFile( - self._getAlignmentRepetitionMdFilename(iteration, 0), - self._getAlignmentMdFilename(iteration) - ) - elif self._getAlignmentRepetitionCount() == 2: + # Run angular distance to bring particles to the closest as possible + alignmentConsensusMds = [emlib.MetaData(self._getAlignmentRepetitionMdFilename(iteration, 0))] + for i in range(1, self._getAlignmentRepetitionCount()): self._runAngularDistance( self._getAlignmentRepetitionMdFilename(iteration, 0), - self._getAlignmentRepetitionMdFilename(iteration, 1), - self._getIterationPath(iteration, 'aligned'), - extrargs=['--compute_average_angle', '--compute_average_shift'] + self._getAlignmentRepetitionMdFilename(iteration, i), + self._getAlignmentConsensusOutputRoot(iteration, i), + extrargs=['--set', 2] ) - # Drop particles far apart - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--query', 'select', 'angleDiff < %f' % angleStep] - self._runMdUtils(args) - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--query', 'select', 'shiftDiff < %f' % shiftStep] - self._runMdUtils(args) - - # Add the missing columns - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--set', 'join', self._getInputParticleMdFilename(), 'itemId'] + args += ['-i', self._getAlignmentConsensusMdFilename(iteration, i)] + args += ['--set', 'join', self._getAlignmentRepetitionMdFilename(iteration, i), 'itemId'] self._runMdUtils(args) - # FIXME - args = [] - args += ['-i', self._getAlignmentMdFilename(iteration)] - args += ['--fill', 'ref3d', 'constant', 1] + alignmentConsensusMds.append(emlib.MetaData(self._getAlignmentRepetitionMdFilename(iteration, i))) - self._runMdUtils(args) - else: - raise NotImplementedError('Alignment consensus only implemented for N=2') + consensusMd = self._computeAlignmentConsensus( + alignmentConsensusMds, + angleStep, + shiftStep + ) + + consensusMd.write(self._getAlignmentMdFilename(iteration)) def intersectInputStep(self, iteration: int): args = [] @@ -738,17 +715,17 @@ def _getAlignmentRepetitionMdFilename(self, iteration: int, repetition: int): def _getAlignmentMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'aligned.xmd') - def _getAlignmentConsensusOutputRoot(self, iteration: int): - return self._getIterationPath(iteration, 'alignment_consensus') + def _getAlignmentConsensusOutputRoot(self, iteration: int, repetition: int): + return self._getIterationPath(iteration, 'alignment_consensus%05d' % repetition) - def _getAlignmentConsensusMdFilename(self, iteration: int): - return self._getAlignmentConsensusOutputRoot(iteration) + '.xmd' + def _getAlignmentConsensusMdFilename(self, iteration: int, repetition: int): + return self._getAlignmentConsensusOutputRoot(iteration, repetition) + '.xmd' - def _getAlignmentConsensusVecDiffHistogramMdFilename(self, iteration: int): - return self._getAlignmentConsensusOutputRoot(iteration) + '_vec_diff_hist.xmd' + def _getAlignmentConsensusVecDiffHistogramMdFilename(self, iteration: int, repetition: int): + return self._getAlignmentConsensusOutputRoot(iteration, repetition) + '_vec_diff_hist.xmd' - def _getAlignmentConsensusShiftDiffHistogramMdFilename(self, iteration: int): - return self._getAlignmentConsensusOutputRoot(iteration) + '_shift_diff_hist.xmd' + def _getAlignmentConsensusShiftDiffHistogramMdFilename(self, iteration: int, repetition: int): + return self._getAlignmentConsensusOutputRoot(iteration, repetition) + '_shift_diff_hist.xmd' def _getInputIntersectionMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'input_intersection.xmd') From 1cc4ef357a1b7f83849c0dcc37a4d1eff10c22f6 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 10 Mar 2023 16:36:04 +0100 Subject: [PATCH 054/104] Updated angular consensus --- .../protocol_reconstruct_swiftres.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 04e9ccb75..24de0695d 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -778,12 +778,18 @@ def _getOutputFscFilename(self, cls: int): def _getTrainingScratchFilename(self): return self._getTmpPath('scratch.bin') - def _averageQuaternions(self, quats: np.ndarray) -> np.ndarray: - s = np.matmul(quats.T, quats) - s /= len(quats) + def _quaternionAverage(self, quaternions: np.ndarray) -> np.ndarray: + s = np.matmul(quaternions.T, quaternions) + s /= len(quaternions) eigenValues, eigenVectors = np.linalg.eig(s) return np.real(eigenVectors[:,np.argmax(eigenValues)]) + def _quaternionDistance(self, + quaternions: np.ndarray, + quaternion: np.ndarray ) -> np.ndarray: + dots = np.dot(quaternions, quaternion) + return np.arccos(2*(dots**2)-1) + def _computeAlignmentConsensus( self, mds: Sequence[emlib.MetaData], maxAngleDiff: float, @@ -812,14 +818,14 @@ def _computeAlignmentConsensus( self, ref3ds[i] = md.getValue(emlib.MDL_REF3D, objId) # Perform a consensus - quaternion, _ = transformations.mean_quaternion(transformations.weighted_tensor(quaternions)) + quaternion = self._quaternionAverage(quaternions) shift = np.mean(shifts, axis=0) ref3d = int(stats.mode(ref3ds).mode) # Check if more than a half agree - angleDiff = np.array(list(map(lambda q : transformations.quaternion_distance(quaternion, q), quaternions))) - #if np.count_nonzero(angleDiff <= maxAngleDiff) <= len(angleDiff) / 2: - # continue + angleDiff = self._quaternionDistance(quaternions, quaternion) + if np.count_nonzero(angleDiff <= maxAngleDiff) <= len(angleDiff) / 2: + continue shiftDiff = np.linalg.norm(shifts-shift, axis=1) if np.count_nonzero(shiftDiff <= maxShiftDiff) <= len(shiftDiff) / 2: @@ -837,7 +843,7 @@ def _computeAlignmentConsensus( self, row.setValue(emlib.MDL_ANGLE_PSI, -np.rad2deg(psi)) row.setValue(emlib.MDL_SHIFT_X, shift[0]) row.setValue(emlib.MDL_SHIFT_Y, shift[1]) - row.setValue(emlib.MDL_ANGLE_DIFF, np.mean(angleDiff)) + row.setValue(emlib.MDL_ANGLE_DIFF, np.rad2deg(np.mean(angleDiff))) row.setValue(emlib.MDL_SHIFT_DIFF, np.mean(shiftDiff)) row.setValue(emlib.MDL_REF3D, ref3d) From 2eeaf15a37482c5cf97860a4be3f67c1f72de0cf Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 13 Mar 2023 10:17:39 +0100 Subject: [PATCH 055/104] Made CTF optional --- .../protocol_reconstruct_swiftres.py | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 24de0695d..dc7bd5d12 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -196,20 +196,31 @@ def convertInputStep(self): def correctCtfStep(self): particles = self.inputParticles.get() + + if particles.hasCTF(): + args = [] + args += ['-i', self._getInputParticleMdFilename()] + args += ['-o', self._getWienerParticleStackFilename()] + args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] + args += ['--keep_input_columns'] + args += ['--sampling_rate', self._getSamplingRate()] + args += ['--pad', '2'] + args += ['--wc', '-1.0'] + if particles.isPhaseFlipped(): + args += ['--phase_flipped'] - args = [] - args += ['-i', self._getInputParticleMdFilename()] - args += ['-o', self._getWienerParticleStackFilename()] - args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] - args += ['--keep_input_columns'] - args += ['--sampling_rate', self._getSamplingRate()] - args += ['--pad', '2'] - args += ['--wc', '-1.0'] - if particles.isPhaseFlipped(): - args += ['--phase_flipped'] + self.runJob('xmipp_ctf_correct_wiener2d', args) + + else: + # TODO When the stack is already in MRC format, simply link + args = [] + args += ['-i', self._getInputParticleMdFilename()] + args += ['-o', self._getWienerParticleStackFilename()] + args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] + args += ['--keep_input_columns'] - self.runJob('xmipp_ctf_correct_wiener2d', args) - + self.runJob('xmipp_image_convert', args, numberOfMpi=1) + def setupIterationStep(self, iteration: int, local: bool): makePath(self._getIterationPath(iteration)) From 4c7703b991118d68db8459291ed49063dba7341c Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 14 Mar 2023 09:30:38 +0100 Subject: [PATCH 056/104] Modified default batch size --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index dc7bd5d12..7c8acd3bf 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -109,8 +109,8 @@ def _defineParams(self, form: Form): help='Maximum number of elements that can be stored in the database ' 'before performing an alignment and flush') form.addParam('batchSize', IntParam, label='Batch size', - default=1024, - help='Batch size used when processing') + default=8192, + help='It is recommended to use powers of 2. Using numbers around 8192 works well') form.addParallelSection(threads=1, mpi=8) From 73fac0072a0b1f6aeab1edb597c1bcb31db22633 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 15 Mar 2023 11:42:22 +0100 Subject: [PATCH 057/104] Added Noise profile views --- .../protocol_reconstruct_swiftres.py | 18 +++ xmipp3/viewers/viewer_reconstruct_swiftres.py | 109 +++++++++++++++++- 2 files changed, 122 insertions(+), 5 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 7c8acd3bf..24d35ef46 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -157,6 +157,8 @@ def _insertProjectSteps(self, iteration: int, prerequisites): # Merge galleries of classes mergeStepIds.append(self._insertFunctionStep('mergeGalleriesStep', iteration, repetition, prerequisites=projectStepIds)) + mergeStepIds.append(self._insertFunctionStep('computeNoiseModelStep', iteration, prerequisites=prerequisites)) + return mergeStepIds def _insertAlignmentSteps(self, iteration: int, local: bool, prerequisites): @@ -335,6 +337,19 @@ def mergeGalleriesStep(self, iteration: int, repetition: int): args += ['--fill', 'ref', 'lineal', 1, 1] self._runMdUtils(args) + def computeNoiseModelStep(self, iteration: int): + args = [] + args += ['-i', self._getIterationInputParticleMdFilename(iteration)] + args += ['-r', self._getIterationInputVolumeFilename(iteration, cls=0)] # TODO for multiple classes + args += ['--oroot', self._getIterationPath(iteration, '')] + self.runJob('xmipp_reconstruct_noise_psd', args, numberOfMpi=1) + + args = [] + args += ['-i', self._getNoiseModelFilename(iteration)] + args += ['-o', self._getWeightsFilename(iteration)] + args += ['--pow', -1] + self.runJob('xmipp_image_operate', args) + def ensembleTrainingSetStep(self, iteration: int): self._mergeMetadata( map(lambda i : self._getGalleryMdFilename(iteration, i), range(self._getAlignmentRepetitionCount())), @@ -714,6 +729,9 @@ def _getGalleryMdFilename(self, iteration: int, repetition: int): def _getTrainingMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'training.xmd') + def _getNoiseModelFilename(self, iteration: int): + return self._getIterationPath(iteration, 'avgNoisePsd.mrc') + def _getWeightsFilename(self, iteration: int): return self._getIterationPath(iteration, 'weights.mrc') diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index c2e9f946b..2adb82218 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -38,8 +38,9 @@ from pwem import emlib from pwem.viewers.views import DataView, ObjectView -from xmipp3.protocols.protocol_reconstruct_swiftres import XmippProtReconstructSwiftres +import xmippLib +from xmipp3.protocols.protocol_reconstruct_swiftres import XmippProtReconstructSwiftres class XmippReconstructSwiftresViewer(ProtocolViewer): @@ -66,6 +67,14 @@ def _defineParams(self, form): form.addParam('showAngleDiffVecDiffHist', LabelParam, label='Display vector difference histogram') form.addParam('showAngleDiffShiftDiffHist', LabelParam, label='Display shift difference histogram') + form.addSection(label='Volumes') + form.addParam('showIterationVolume', IntParam, label='Display iteration volume', + default=0) + + form.addSection(label='Noise model') + form.addParam('showNoiseModel', LabelParam, label='Display radial noise model profile') + form.addParam('showWeights', LabelParam, label='Display radial weight profile') + form.addSection(label='Classification') form.addParam('showClassMigration', LabelParam, label='Display class migration diagram', default=0.143) @@ -80,7 +89,12 @@ def _getVisualizeDict(self): 'showAngleDiffVecDiffHist': self._showAngleDiffVecDiffHistogram, 'showAngleDiffShiftDiffHist': self._showAngleDiffShiftDiffHistogram, - 'showClassMigration': self._showClassMigration + 'showIterationVolume': self._showIterationVolume, + + 'showNoiseModel': self._showNoiseModel, + 'showWeights': self._showWeights, + + #'showClassMigration': self._showClassMigration } @@ -100,6 +114,9 @@ def _getAlignmentMdFilename(self, iteration: int): def _getFscFilename(self, iteration: int, cls: int): return self.protocol._getFscFilename(iteration, cls) + def _getFilteredReconstructionFilename(self, iteration: int, cls: int): + return self.protocol._getFilteredReconstructionFilename(iteration, cls) + def _getAngleDiffMdFilename(self, iteration: int): return self.protocol._getAngleDiffMdFilename(iteration) @@ -109,6 +126,12 @@ def _getAngleDiffVecDiffHistogramMdFilename(self, iteration: int): def _getAngleDiffShiftDiffHistogramMdFilename(self, iteration: int): return self.protocol._getAngleDiffShiftDiffHistogramMdFilename(iteration) + def _getNoiseModelFilename(self, iteration: int): + return self.protocol._getNoiseModelFilename(iteration) + + def _getWeightsFilename(self, iteration: int): + return self.protocol._getWeightsFilename(iteration) + def _readFsc(self, fscMd: emlib.MetaData) -> np.ndarray: values = [] for objId in fscMd: @@ -138,12 +161,22 @@ def _iterFscMdClassFilenames(self, cls: int): def _iterAngleDiffVecDiffHistMdFilenames(self): return self._iterFilenamesUntilNotExist( - map(lambda it : self._getAngleDiffVecDiffHistogramMdFilename(it), itertools.count()) + map(self._getAngleDiffVecDiffHistogramMdFilename, itertools.count()) ) def _iterAngleDiffShiftDiffHistMdFilenames(self): return self._iterFilenamesUntilNotExist( - map(lambda it : self._getAngleDiffShiftDiffHistogramMdFilename(it), itertools.count()) + map(self._getAngleDiffShiftDiffHistogramMdFilename, itertools.count()) + ) + + def _iterNoiseModelFilenames(self): + return self._iterFilenamesUntilNotExist( + map(self._getNoiseModelFilename, itertools.count()) + ) + + def _iterWeightsFilenames(self): + return self._iterFilenamesUntilNotExist( + map(self._getWeightsFilename, itertools.count()) ) def _readFscCutoff(self, cls: int, cutoff: float) -> np.ndarray: @@ -207,9 +240,12 @@ def _computeClassMigrationElements(self, alignmentFilenames: Iterator[str]) -> n for iteration, dstAlignmentFn in enumerate(alignmentFilenames, start=1): dstAlignmentMd = emlib.MetaData(dstAlignmentFn) + # Obtain common elements srcAlignmentMd.intersection(dstAlignmentMd, emlib.MDL_ITEM_ID) - # TODO + + #points = self._computeClassPoints(dstAlignmentMd) + #points = np.c_[np.full(len(points), iteration), points] # Save for next srcAlignmentMd = dstAlignmentMd @@ -230,6 +266,24 @@ def _readAngleDiffShiftHistogram(self, filename): return diff, count + def _computePsdRadialProfile(self, psd: np.ndarray, n: int = 128) -> np.ndarray: + result = np.empty(n) + + # Compute the frequency grid + freqX = np.fft.rfftfreq(len(psd)) + freqY = np.fft.fftfreq(len(psd)) + freq2 = freqX**2 + (freqY**2)[None].T + + # Select by bands + limits = np.linspace(0, 0.5, n+1) + limits2 = limits**2 + for i in range(len(result)): + low2, high2 = limits2[i], limits2[i+1] + mask = np.logical_and(low2 <= freq2, freq2 <= high2) + result[i] = np.average(psd[mask]) + + return result + # ---------------------------- SHOW functions ------------------------------ def _showIterationFsc(self, e): fig, ax = plt.subplots() @@ -324,6 +378,51 @@ def _showAngleDiffShiftDiffHistogram(self, e): return [fig] + def _showIterationVolume(self, e): + iteration = int(self.showIterationVolume) + filename = self._getFilteredReconstructionFilename(iteration, 0) + return DataView(filename) + + def _showNoiseModel(self, e): + fig, ax = plt.subplots() + + psd = xmippLib.Image() + freq = np.linspace(0, 0.5, 128)[:-1] + freq += freq[1] / 2 + for iteration, filename in enumerate(self._iterNoiseModelFilenames()): + psd.read(filename) + profile = self._computePsdRadialProfile(psd.getData(), len(freq)) + label = f'Iteration {iteration}' + ax.plot(freq, profile, label=label) + + ax.set_title('Noise model') + ax.set_xlabel('Resolution (1/A)') + ax.set_ylabel('$\sigma^2$') + ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) + ax.legend() + + return [fig] + + def _showWeights(self, e): + fig, ax = plt.subplots() + + psd = xmippLib.Image() + freq = np.linspace(0, 0.5, 128)[:-1] + freq += freq[1] / 2 + for iteration, filename in enumerate(self._iterWeightsFilenames()): + psd.read(filename) + profile = self._computePsdRadialProfile(psd.getData(), len(freq)) + label = f'Iteration {iteration}' + ax.plot(freq, profile, label=label) + + ax.set_title('Weights') + ax.set_xlabel('Resolution (1/A)') + ax.set_ylabel('Weight') + ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) + ax.legend() + + return [fig] + def _showClassMigration(self, e): fig, ax = plt.subplots() From d0b29e855dd1681d2efd3a2bbe92e3472c01f583 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 15 Mar 2023 12:48:07 +0100 Subject: [PATCH 058/104] Added parameters to noise model generation --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 24d35ef46..339fd4fc0 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -342,6 +342,8 @@ def computeNoiseModelStep(self, iteration: int): args += ['-i', self._getIterationInputParticleMdFilename(iteration)] args += ['-r', self._getIterationInputVolumeFilename(iteration, cls=0)] # TODO for multiple classes args += ['--oroot', self._getIterationPath(iteration, '')] + args += ['--padding', 2.0] + args += ['--max_resolution', 1.0] # TODO self.runJob('xmipp_reconstruct_noise_psd', args, numberOfMpi=1) args = [] From cc773add71ca4db38f7d3c51edd783ec404804a7 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 21 Mar 2023 11:46:30 +0100 Subject: [PATCH 059/104] Added class size plot --- xmipp3/viewers/viewer_reconstruct_swiftres.py | 267 +++++++++++------- 1 file changed, 160 insertions(+), 107 deletions(-) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index 2adb82218..5c95c0d92 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -53,48 +53,51 @@ class XmippReconstructSwiftresViewer(ProtocolViewer): # --------------------------- DEFINE param functions ----------------------- def _defineParams(self, form): - form.addSection(label='FSC') - form.addParam('showIterationFsc', IntParam, label='Display iteration FSC', - default=0 ) - form.addParam('showClassFsc', IntParam, label='Display class FSC', - default=0) - form.addParam('showFscCutoff', FloatParam, label='Display FSC cutoff', - default=0.143) - - form.addSection(label='Angular difference') + form.addSection(label='Noise model') + form.addParam('showNoiseModel', LabelParam, label='Display radial noise model profile') + form.addParam('showWeights', LabelParam, label='Display radial weight profile') + + form.addSection(label='Classification') + form.addParam('showClassMigration', LabelParam, label='Display class migration diagram') + form.addParam('showClassSizes', LabelParam, label='Display class sizes') + + form.addSection(label='Angular difference from previous iteration') form.addParam('showAngleDiffMetadata', IntParam, label='Display iteration angular difference metadata', default=0) form.addParam('showAngleDiffVecDiffHist', LabelParam, label='Display vector difference histogram') form.addParam('showAngleDiffShiftDiffHist', LabelParam, label='Display shift difference histogram') - form.addSection(label='Volumes') + form.addSection(label='Reconstruction') form.addParam('showIterationVolume', IntParam, label='Display iteration volume', default=0) - form.addSection(label='Noise model') - form.addParam('showNoiseModel', LabelParam, label='Display radial noise model profile') - form.addParam('showWeights', LabelParam, label='Display radial weight profile') - - form.addSection(label='Classification') - form.addParam('showClassMigration', LabelParam, label='Display class migration diagram', + form.addSection(label='FSC') + form.addParam('showIterationFsc', IntParam, label='Display iteration FSC', + default=0 ) + form.addParam('showClassFsc', IntParam, label='Display class FSC', + default=0) + form.addParam('showFscCutoff', FloatParam, label='Display FSC cutoff', default=0.143) + def _getVisualizeDict(self): return { - 'showIterationFsc': self._showIterationFsc, - 'showClassFsc': self._showClassFsc, - 'showFscCutoff': self._showFscCutoff, + 'showNoiseModel': self._showNoiseModel, + 'showWeights': self._showWeights, + 'showClassMigration': self._showClassMigration, + 'showClassSizes': self._showClassSizes, + 'showAngleDiffMetadata': self._showAngleDiffMetadata, 'showAngleDiffVecDiffHist': self._showAngleDiffVecDiffHistogram, 'showAngleDiffShiftDiffHist': self._showAngleDiffShiftDiffHistogram, 'showIterationVolume': self._showIterationVolume, + + 'showIterationFsc': self._showIterationFsc, + 'showClassFsc': self._showClassFsc, + 'showFscCutoff': self._showFscCutoff, - 'showNoiseModel': self._showNoiseModel, - 'showWeights': self._showWeights, - - #'showClassMigration': self._showClassMigration } @@ -148,7 +151,21 @@ def _iterFilenamesUntilNotExist(self, filenames: Iterable[str]): else: break + + def _iterMetadatas(self, filenames: Iterable[str]) -> emlib.MetaData: + md = emlib.MetaData() + for fn in filenames: + md.read(fn) + yield md + def _iterAlignmentMdFilenames(self): + return self._iterFilenamesUntilNotExist( + map(self._getAlignmentMdFilename, itertools.count()) + ) + + def _iterAlignmentMds(self): + return self._iterMetadatas(self._iterAlignmentMdFilenames()) + def _iterFscMdIterationFilenames(self, it: int): return self._iterFilenamesUntilNotExist( map(lambda cls : self._getFscFilename(it, cls), itertools.count()) @@ -202,6 +219,27 @@ def _computeFscTickLabels(self, ticks: np.ndarray) -> np.ndarray: labels = list(map('1/{0:.2f}'.format, samplingRate/ticks)) return labels + def _computeIterationClassSizes(self, alignmentMd: emlib.MetaData, label: int = emlib.MDL_REF3D) -> dict: + classIds = alignmentMd.getColumnValues(label) + return collections.Counter(classIds) + + def _computeClassSizes(self, alignmentMds: Iterable[emlib.MetaData], label: int = emlib.MDL_REF3D) -> dict: + result = dict() + + for it, alignmentMd in enumerate(alignmentMds): + counts = self._computeIterationClassSizes(alignmentMd, label) + + # Process existing elements + for id in result.keys(): + result[id].append(counts.get(id, 0)) + + # Process new elements + for id, count in counts.items(): + if id not in result: + result[id] = [0]*it + [count] + + return { classId: np.array(counts) for classId, counts in result.items() } + def _computeClassMigrations(self, srcMd: emlib.MetaData, dstMd: emlib.MetaData) -> dict: result = {} @@ -226,11 +264,6 @@ def _computeIterationClassMigrationSegments(self, migrations: dict) -> Tuple[np. segments[i,:] = (srcClass, dstClass) return segments, counts - - def _computeClassPoints(self, alignmentMd: emlib.MetaData) -> np.ndarray: - class3d = alignmentMd.getColumnValues(emlib.MDL_REF3D) - freq = collections.Counter(class3d) - return np.array(list(freq.items())) def _computeClassMigrationElements(self, alignmentFilenames: Iterator[str]) -> np.ndarray: points = [] @@ -285,64 +318,84 @@ def _computePsdRadialProfile(self, psd: np.ndarray, n: int = 128) -> np.ndarray: return result # ---------------------------- SHOW functions ------------------------------ - def _showIterationFsc(self, e): + def _showNoiseModel(self, e): fig, ax = plt.subplots() - - it = int(self.showIterationFsc) - for cls, fscFn in enumerate(self._iterFscMdIterationFilenames(it), start=1): - fscMd = emlib.MetaData(fscFn) - fsc = self._readFsc(fscMd) - label = f'Class {cls}' - ax.plot(fsc[:,0], fsc[:,1], label=label) - ax.axhline(0.5, color='black', linestyle='--') - ax.axhline(0.143, color='black', linestyle='--') - ax.set_title('Class FSC') + psd = xmippLib.Image() + freq = np.linspace(0, 0.5, 128)[:-1] + freq += freq[1] / 2 + for iteration, filename in enumerate(self._iterNoiseModelFilenames()): + psd.read(filename) + profile = self._computePsdRadialProfile(psd.getData(), len(freq)) + label = f'Iteration {iteration}' + ax.plot(freq, profile, label=label) + + ax.set_title('Noise model') ax.set_xlabel('Resolution (1/A)') - ax.set_ylabel('FSC') + ax.set_ylabel('$\sigma^2$') ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) ax.legend() - + return [fig] - def _showClassFsc(self, e): + def _showWeights(self, e): fig, ax = plt.subplots() - - cls = int(self.showClassFsc) - for it, fscFn in enumerate(self._iterFscMdClassFilenames(cls), start=1): - fscMd = emlib.MetaData(fscFn) - fsc = self._readFsc(fscMd) - label = f'Iteration {it}' - ax.plot(fsc[:,0], fsc[:,1], label=label) - ax.axhline(0.5, color='black', linestyle='--') - ax.axhline(0.143, color='black', linestyle='--') - ax.set_title('Class FSC') + psd = xmippLib.Image() + freq = np.linspace(0, 0.5, 128)[:-1] + freq += freq[1] / 2 + for iteration, filename in enumerate(self._iterWeightsFilenames()): + psd.read(filename) + profile = self._computePsdRadialProfile(psd.getData(), len(freq)) + label = f'Iteration {iteration}' + ax.plot(freq, profile, label=label) + + ax.set_title('Weights') ax.set_xlabel('Resolution (1/A)') - ax.set_ylabel('FSC') + ax.set_ylabel('Weight') ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) ax.legend() - + return [fig] - - def _showFscCutoff(self, e): + + def _showClassMigration(self, e): fig, ax = plt.subplots() - cutoff = float(self.showFscCutoff) - for cls in range(self._getClassCount()): - y = self._readFscCutoff(cls, cutoff) - x = np.arange(len(y)) - label = f'Class {cls}' - ax.plot(x, y, label=label) + it = self._iterAlignmentMdFilenames() + segments, counts = self._computeClassMigrationSegments(it) + lines = matplotlib.collections.LineCollection(segments, array=counts) - ax.set_title('Resolution') + it = self._iterAlignmentMdFilenames() + points, sizes = self._computeClassSizePoints(it) + + ax.add_collection(lines) + sc = ax.scatter(points[:,0], points[:,1], c=sizes) ax.set_xlabel('Iteration') - ax.set_ylabel('Resolution (A)') - ax.set_ylim(bottom=0) + ax.set_ylabel('Class') + fig.colorbar(sc, ax=ax) + + return [fig] + + def _showClassSizes(self, e): + fig, ax = plt.subplots() + + sizes = self._computeClassSizes(self._iterAlignmentMds()) + bottom = None + for label, counts in sizes.items(): + x = np.arange(len(counts)) + ax.bar(x=x, height=counts, bottom=bottom, label=label) + + if bottom is None: + bottom = counts.copy() + else: + bottom += counts + + ax.set_xlabel('Iteration') + ax.set_ylabel('Class size') ax.legend() - + return [fig] - + def _showAngleDiffMetadata(self, e): mdFilename = self._getAngleDiffMdFilename(int(self.showAngleDiffMetadata)) v = DataView(mdFilename) @@ -383,61 +436,61 @@ def _showIterationVolume(self, e): filename = self._getFilteredReconstructionFilename(iteration, 0) return DataView(filename) - def _showNoiseModel(self, e): + + def _showIterationFsc(self, e): fig, ax = plt.subplots() + + it = int(self.showIterationFsc) + for cls, fscFn in enumerate(self._iterFscMdIterationFilenames(it), start=1): + fscMd = emlib.MetaData(fscFn) + fsc = self._readFsc(fscMd) + label = f'Class {cls}' + ax.plot(fsc[:,0], fsc[:,1], label=label) + ax.axhline(0.5, color='black', linestyle='--') + ax.axhline(0.143, color='black', linestyle='--') - psd = xmippLib.Image() - freq = np.linspace(0, 0.5, 128)[:-1] - freq += freq[1] / 2 - for iteration, filename in enumerate(self._iterNoiseModelFilenames()): - psd.read(filename) - profile = self._computePsdRadialProfile(psd.getData(), len(freq)) - label = f'Iteration {iteration}' - ax.plot(freq, profile, label=label) - - ax.set_title('Noise model') + ax.set_title('Class FSC') ax.set_xlabel('Resolution (1/A)') - ax.set_ylabel('$\sigma^2$') + ax.set_ylabel('FSC') ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) ax.legend() - + return [fig] - def _showWeights(self, e): + def _showClassFsc(self, e): fig, ax = plt.subplots() + + cls = int(self.showClassFsc) + for it, fscFn in enumerate(self._iterFscMdClassFilenames(cls), start=1): + fscMd = emlib.MetaData(fscFn) + fsc = self._readFsc(fscMd) + label = f'Iteration {it}' + ax.plot(fsc[:,0], fsc[:,1], label=label) + ax.axhline(0.5, color='black', linestyle='--') + ax.axhline(0.143, color='black', linestyle='--') - psd = xmippLib.Image() - freq = np.linspace(0, 0.5, 128)[:-1] - freq += freq[1] / 2 - for iteration, filename in enumerate(self._iterWeightsFilenames()): - psd.read(filename) - profile = self._computePsdRadialProfile(psd.getData(), len(freq)) - label = f'Iteration {iteration}' - ax.plot(freq, profile, label=label) - - ax.set_title('Weights') + ax.set_title('Class FSC') ax.set_xlabel('Resolution (1/A)') - ax.set_ylabel('Weight') + ax.set_ylabel('FSC') ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) ax.legend() - + return [fig] - - def _showClassMigration(self, e): + + def _showFscCutoff(self, e): fig, ax = plt.subplots() - it = self._iterAlignmentMdFilenames() - segments, counts = self._computeClassMigrationSegments(it) - lines = matplotlib.collections.LineCollection(segments, array=counts) + cutoff = float(self.showFscCutoff) + for cls in range(self._getClassCount()): + y = self._readFscCutoff(cls, cutoff) + x = np.arange(len(y)) + label = f'Class {cls}' + ax.plot(x, y, label=label) - it = self._iterAlignmentMdFilenames() - points, sizes = self._computeClassSizePoints(it) - - ax.add_collection(lines) - sc = ax.scatter(points[:,0], points[:,1], c=sizes) + ax.set_title('Resolution') ax.set_xlabel('Iteration') - ax.set_ylabel('Class') - fig.colorbar(sc, ax=ax) - + ax.set_ylabel('Resolution (A)') + ax.set_ylim(bottom=0) + ax.legend() + return [fig] - \ No newline at end of file From b1a69401479636891d3dd85cefc1cce57220e9a6 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 22 Mar 2023 12:11:31 +0100 Subject: [PATCH 060/104] Added angular distribution plots --- .../protocol_reconstruct_swiftres.py | 24 ++-- xmipp3/viewers/viewer_reconstruct_swiftres.py | 110 +++++++++++++++--- 2 files changed, 107 insertions(+), 27 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 339fd4fc0..9bf52885a 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -176,7 +176,7 @@ def _insertAlignmentSteps(self, iteration: int, local: bool, prerequisites): def _insertReconstructSteps(self, iteration: int, cls: int, prerequisites): selectAlignmentStepId = self._insertFunctionStep('selectAlignmentStep', iteration, cls, prerequisites=prerequisites) - #compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[selectAlignmentStepId]) + compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[selectAlignmentStepId]) #computeWeightsStepId = self._insertFunctionStep('computeWeightsStep', iteration, cls, prerequisites=[compareReprojectionStepId]) #filterByWeightsStepId = self._insertFunctionStep('filterByWeightsStep', iteration, cls, prerequisites=[computeWeightsStepId]) splitStepId = self._insertFunctionStep('splitStep', iteration, cls, prerequisites=[selectAlignmentStepId]) @@ -464,7 +464,8 @@ def alignmentConsensusStep(self, iteration: int): consensusMd = self._computeAlignmentConsensus( alignmentConsensusMds, angleStep, - shiftStep + shiftStep, + np.inf ) consensusMd.write(self._getAlignmentMdFilename(iteration)) @@ -824,7 +825,8 @@ def _quaternionDistance(self, def _computeAlignmentConsensus( self, mds: Sequence[emlib.MetaData], maxAngleDiff: float, - maxShiftDiff: float ) -> emlib.MetaData: + maxShiftDiff: float, + maxCost: float ) -> emlib.MetaData: if len(mds) > 1: resultMd = emlib.MetaData() @@ -833,6 +835,7 @@ def _computeAlignmentConsensus( self, quaternions = np.empty((len(mds), 4)) shifts = np.empty((len(mds), 2)) ref3ds = np.empty(len(mds), dtype=int) + costs = np.empty(len(mds)) row = emlib.metadata.Row() for objIds in zip(*mds): @@ -847,24 +850,27 @@ def _computeAlignmentConsensus( self, shifts[i,:] = ( md.getValue(emlib.MDL_SHIFT_X, objId), md.getValue(emlib.MDL_SHIFT_Y, objId) ) ref3ds[i] = md.getValue(emlib.MDL_REF3D, objId) + costs[i] = md.getValue(emlib.MDL_COST, objId) - # Perform a consensus - quaternion = self._quaternionAverage(quaternions) - shift = np.mean(shifts, axis=0) - ref3d = int(stats.mode(ref3ds).mode) - # Check if more than a half agree + quaternion = self._quaternionAverage(quaternions) angleDiff = self._quaternionDistance(quaternions, quaternion) if np.count_nonzero(angleDiff <= maxAngleDiff) <= len(angleDiff) / 2: continue + shift = np.mean(shifts, axis=0) shiftDiff = np.linalg.norm(shifts-shift, axis=1) if np.count_nonzero(shiftDiff <= maxShiftDiff) <= len(shiftDiff) / 2: continue + ref3d = int(stats.mode(ref3ds).mode) if np.count_nonzero(ref3ds == ref3d) <= len(ref3ds) / 2: continue + cost = np.median(costs) + if cost > maxCost: + continue + # All checks succeeded. Elaborate output row.readFromMd(mds[0], objIds[0]) @@ -877,6 +883,7 @@ def _computeAlignmentConsensus( self, row.setValue(emlib.MDL_ANGLE_DIFF, np.rad2deg(np.mean(angleDiff))) row.setValue(emlib.MDL_SHIFT_DIFF, np.mean(shiftDiff)) row.setValue(emlib.MDL_REF3D, ref3d) + row.setValue(emlib.MDL_COST, cost) assert(len(mds) > 1) row.setValue(emlib.MDL_ANGLE_PSI+1, mds[0].getValue(emlib.MDL_ANGLE_PSI, objIds[0])) @@ -890,6 +897,7 @@ def _computeAlignmentConsensus( self, row.setValue(emlib.MDL_SHIFT_Y+1, mds[0].getValue(emlib.MDL_SHIFT_Y, objIds[0])) row.setValue(emlib.MDL_SHIFT_Y+2, mds[1].getValue(emlib.MDL_SHIFT_Y, objIds[1])) + row.writeToMd(resultMd, resultMd.addObject()) return resultMd diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index 5c95c0d92..bb7131923 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -29,14 +29,16 @@ import itertools import collections import numpy as np -import matplotlib.collections +from matplotlib.collections import LineCollection import matplotlib.pyplot as plt +import matplotlib.cm as cm from pyworkflow.viewer import ProtocolViewer, DESKTOP_TKINTER, WEB_DJANGO from pyworkflow.protocol.params import LabelParam, IntParam, FloatParam from pwem import emlib from pwem.viewers.views import DataView, ObjectView +from pwem.viewers import EmPlotter import xmippLib @@ -68,6 +70,12 @@ def _defineParams(self, form): form.addParam('showAngleDiffShiftDiffHist', LabelParam, label='Display shift difference histogram') form.addSection(label='Reconstruction') + form.addParam('showIterationAngularDistribution', IntParam, label='Display iteration angular distribution', + default=0) + form.addParam('showIterationAngularDistribution3d', IntParam, label='Display iteration angular distribution 3D', + default=0) + form.addParam('showIterationShiftHistogram', IntParam, label='Display iteration shift histogram', + default=0) form.addParam('showIterationVolume', IntParam, label='Display iteration volume', default=0) @@ -92,6 +100,9 @@ def _getVisualizeDict(self): 'showAngleDiffVecDiffHist': self._showAngleDiffVecDiffHistogram, 'showAngleDiffShiftDiffHist': self._showAngleDiffShiftDiffHistogram, + 'showIterationAngularDistribution': self._showIterationAngularDistribution, + 'showIterationAngularDistribution3d': self._showIterationAngularDistribution3d, + 'showIterationShiftHistogram': self._showIterationShiftHistogram, 'showIterationVolume': self._showIterationVolume, 'showIterationFsc': self._showIterationFsc, @@ -102,6 +113,9 @@ def _getVisualizeDict(self): # --------------------------- UTILS functions ------------------------------ + def _getIterationParametersFilename(self, iteration: int): + return self.protocol._getIterationParametersFilename(iteration) + def _getIterationCount(self) -> int: return self.protocol._getIterationCount() @@ -260,14 +274,12 @@ def _computeIterationClassMigrationSegments(self, migrations: dict) -> Tuple[np. counts = np.array(list(migrations.values())) # Write the y values form migrations - for i, (srcClass, dstClass) in enumerate(migrations.keys()): + for i, (srcClass, dstClass) in enumerate(migrations.items()): segments[i,:] = (srcClass, dstClass) return segments, counts - def _computeClassMigrationElements(self, alignmentFilenames: Iterator[str]) -> np.ndarray: - points = [] - + def _computeClassMigrationElements(self, alignmentFilenames: Iterator[str]) -> Tuple[LineCollection, np.ndarray]: srcAlignmentFn = next(alignmentFilenames) srcAlignmentMd = emlib.MetaData(srcAlignmentFn) for iteration, dstAlignmentFn in enumerate(alignmentFilenames, start=1): @@ -276,15 +288,15 @@ def _computeClassMigrationElements(self, alignmentFilenames: Iterator[str]) -> n # Obtain common elements srcAlignmentMd.intersection(dstAlignmentMd, emlib.MDL_ITEM_ID) - - #points = self._computeClassPoints(dstAlignmentMd) - #points = np.c_[np.full(len(points), iteration), points] + migrations = self._computeClassMigrations(srcAlignmentMd, dstAlignmentMd) + sizes = self._computeIterationClassSizes(dstAlignmentMd) + segments, counts = self._computeIterationClassMigrationSegments(migrations) # Save for next srcAlignmentMd = dstAlignmentMd - - return np.concatenate(points) - + + #TODO + def _readAngleDiffVecHistogram(self, filename): md = emlib.MetaData(filename) diff = md.getColumnValues(emlib.MDL_ANGLE_DIFF) @@ -361,15 +373,10 @@ def _showWeights(self, e): def _showClassMigration(self, e): fig, ax = plt.subplots() - it = self._iterAlignmentMdFilenames() - segments, counts = self._computeClassMigrationSegments(it) - lines = matplotlib.collections.LineCollection(segments, array=counts) - - it = self._iterAlignmentMdFilenames() - points, sizes = self._computeClassSizePoints(it) - + lines, points = self._computeClassMigrationElements() + ax.add_collection(lines) - sc = ax.scatter(points[:,0], points[:,1], c=sizes) + sc = ax.scatter(points[:,0], points[:,1], c=points[:,2]) ax.set_xlabel('Iteration') ax.set_ylabel('Class') fig.colorbar(sc, ax=ax) @@ -436,6 +443,70 @@ def _showIterationVolume(self, e): filename = self._getFilteredReconstructionFilename(iteration, 0) return DataView(filename) + def _showIterationAngularDistribution(self, e): + iteration = int(self.showIterationAngularDistribution) + view = EmPlotter(x=1, y=1, mainTitle="Iteration %d" % iteration, windowTitle="Angular distribution") + axis = view.plotAngularDistributionFromMd(self._getAlignmentMdFilename(iteration), '', histogram=True) + view.getFigure().colorbar(axis) + return [view] + + def _showIterationAngularDistribution3d(self, e): + fig = plt.figure() + ax = fig.add_subplot(111, projection='3d') + + # Read data + iteration = int(self.showIterationAngularDistribution3d) + md = emlib.MetaData(self._getAlignmentMdFilename(iteration)) + rot = np.deg2rad(md.getColumnValues(emlib.MDL_ANGLE_ROT)) + tilt = np.deg2rad(md.getColumnValues(emlib.MDL_ANGLE_TILT)) + + # Make histogram + N = 32 + heatmap, rotEdges, tiltEdges = np.histogram2d( + x=rot, + y=tilt, + density=True, + range=[[-np.pi, +np.pi], [0.0, np.pi]], + bins=N + ) + + # Cartesian coordinates in unit sphere + rotEdges, tiltEdges = np.meshgrid(rotEdges, tiltEdges) + x = np.sin(tiltEdges) * np.cos(rotEdges) + y = np.sin(tiltEdges) * np.sin(rotEdges) + z = np.cos(tiltEdges) + + # Plot surface + ax.plot_surface(x, y, z, rstride=1, cstride=1, facecolors=cm.plasma(heatmap)) + ax.set_title('Angular distribution') + + return [fig] + + def _showIterationShiftHistogram(self, e): + fig, ax = plt.subplots() + + iteration = int(self.showIterationShiftHistogram) + md = emlib.MetaData(self._getAlignmentMdFilename(iteration)) + + shiftX = md.getColumnValues(emlib.MDL_SHIFT_X) + shiftY = md.getColumnValues(emlib.MDL_SHIFT_Y) + + N = 128 + heatmap, xEdges, yEdges = np.histogram2d( + x=shiftX, + y=shiftY, + density=True, + bins=N + ) + + image = ax.imshow(heatmap, extent=(xEdges[0], xEdges[-1], yEdges[0], yEdges[-1])) + ax.set_title('Shift histogram') + ax.set_xlabel('Shift X [px]') + ax.set_ylabel('Shift Y [px]') + fig.colorbar(image) + + return [fig] + def _showIterationFsc(self, e): fig, ax = plt.subplots() @@ -494,3 +565,4 @@ def _showFscCutoff(self, e): ax.legend() return [fig] + From c3cbfa4635715414ca1199aa53713976e9976398 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 22 Mar 2023 12:42:08 +0100 Subject: [PATCH 061/104] Removed extra linebreak --- xmipp3/viewers/viewer_reconstruct_swiftres.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index bb7131923..98b8b7354 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -507,7 +507,6 @@ def _showIterationShiftHistogram(self, e): return [fig] - def _showIterationFsc(self, e): fig, ax = plt.subplots() From e13701662942daac7c16ee1536bbcd4c3e73fca8 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 23 Mar 2023 11:27:50 +0100 Subject: [PATCH 062/104] Fixed warning --- .../protocol_reconstruct_swiftres.py | 19 ++-- xmipp3/viewers/viewer_reconstruct_swiftres.py | 94 ++++++++++++------- 2 files changed, 70 insertions(+), 43 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 9bf52885a..96398f01e 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -124,11 +124,11 @@ def _insertAllSteps(self): lastIds = [correctCtfStepId] for i in range(self._getIterationCount()): - lastIds = self._insertIterationSteps(i, False, prerequisites=lastIds) + lastIds = self._insertIterationSteps(i, 0, prerequisites=lastIds) self._insertFunctionStep('createOutputStep', prerequisites=lastIds) - def _insertIterationSteps(self, iteration: int, local: bool, prerequisites): + def _insertIterationSteps(self, iteration: int, local: int, prerequisites): setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, local, prerequisites=prerequisites) #ctfGroupStepId = self._insertFunctionStep('ctfGroupStep', iteration, prerequisites=[setupIterationStepId]) projectIds = self._insertProjectSteps(iteration, prerequisites=[setupIterationStepId]) @@ -161,7 +161,7 @@ def _insertProjectSteps(self, iteration: int, prerequisites): return mergeStepIds - def _insertAlignmentSteps(self, iteration: int, local: bool, prerequisites): + def _insertAlignmentSteps(self, iteration: int, local: int, prerequisites): ensembleTrainingSetStepId = self._insertFunctionStep('ensembleTrainingSetStep', iteration, prerequisites=prerequisites) trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=[ensembleTrainingSetStepId]) @@ -223,7 +223,7 @@ def correctCtfStep(self): self.runJob('xmipp_image_convert', args, numberOfMpi=1) - def setupIterationStep(self, iteration: int, local: bool): + def setupIterationStep(self, iteration: int, local: int): makePath(self._getIterationPath(iteration)) for cls in range(self._getClassCount()): @@ -256,6 +256,9 @@ def setupIterationStep(self, iteration: int, local: bool): resolution = float(self.initialResolution) + if local > 0: + raise NotImplementedError('Local searches are not implemented') + imageSize = self._getImageSize() frequency = self._getSamplingRate() / resolution maxPsi = self._getIterationMaxPsi(iteration) @@ -388,7 +391,7 @@ def trainDatabaseStep(self, iteration: int): env['LD_LIBRARY_PATH'] = '' # Torch does not like it self.runJob('xmipp_train_database', args, numberOfMpi=1, env=env) - def alignStep(self, iteration: int, repetition: int, local: bool): + def alignStep(self, iteration: int, repetition: int, local: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) imageSize = self._getImageSize() maxFrequency = md.getValue(emlib.MDL_RESOLUTION_FREQREAL, 1) @@ -416,7 +419,7 @@ def alignStep(self, iteration: int, repetition: int, local: bool): args += ['--max_size', self.databaseMaximumSize] if self.useGpu: args += ['--device', 'cuda:0'] # TODO select - if local: + if local > 0: args += ['--local_shift', '--local_psi'] env = self.getCondaEnv() @@ -665,6 +668,9 @@ def createOutputStep(self): def _getIterationCount(self) -> int: return int(self.numberOfIterations) + def _getCompletedIterationCount(self) -> int: + return self._getIterationCount() # TODO + def _getAlignmentRepetitionCount(self) -> int: return int(self.numberOfAlignmentRepetitions) @@ -680,7 +686,6 @@ def _getSamplingRate(self) -> float: def _getImageSize(self) -> int: return int(self.inputParticles.get().getXDim()) - def _getIterationPath(self, iteration: int, *paths): return self._getExtraPath('iteration_%04d' % iteration, *paths) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index 98b8b7354..e49face90 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -55,6 +55,12 @@ class XmippReconstructSwiftresViewer(ProtocolViewer): # --------------------------- DEFINE param functions ----------------------- def _defineParams(self, form): + form.addSection(label='Settings') + form.addParam('iteration', IntParam, label='Iteration', default=-1, + help='Iteration to be visualized. Zero based. Use -1 for last one') + form.addParam('classId', IntParam, label='Class', default=1, + help='Class to be visualized. One based') + form.addSection(label='Noise model') form.addParam('showNoiseModel', LabelParam, label='Display radial noise model profile') form.addParam('showWeights', LabelParam, label='Display radial weight profile') @@ -64,26 +70,19 @@ def _defineParams(self, form): form.addParam('showClassSizes', LabelParam, label='Display class sizes') form.addSection(label='Angular difference from previous iteration') - form.addParam('showAngleDiffMetadata', IntParam, label='Display iteration angular difference metadata', - default=0) + form.addParam('showAngleDiffMetadata', LabelParam, label='Display iteration angular difference metadata') form.addParam('showAngleDiffVecDiffHist', LabelParam, label='Display vector difference histogram') form.addParam('showAngleDiffShiftDiffHist', LabelParam, label='Display shift difference histogram') form.addSection(label='Reconstruction') - form.addParam('showIterationAngularDistribution', IntParam, label='Display iteration angular distribution', - default=0) - form.addParam('showIterationAngularDistribution3d', IntParam, label='Display iteration angular distribution 3D', - default=0) - form.addParam('showIterationShiftHistogram', IntParam, label='Display iteration shift histogram', - default=0) - form.addParam('showIterationVolume', IntParam, label='Display iteration volume', - default=0) + form.addParam('showIterationAngularDistribution', LabelParam, label='Display iteration angular distribution') + form.addParam('showIterationAngularDistribution3d', LabelParam, label='Display iteration angular distribution 3D') + form.addParam('showIterationShiftHistogram', LabelParam, label='Display iteration shift histogram') + form.addParam('showIterationVolume', LabelParam, label='Display iteration volume') form.addSection(label='FSC') - form.addParam('showIterationFsc', IntParam, label='Display iteration FSC', - default=0 ) - form.addParam('showClassFsc', IntParam, label='Display class FSC', - default=0) + form.addParam('showIterationFsc', LabelParam, label='Display iteration FSC') + form.addParam('showClassFsc', LabelParam, label='Display class FSC') form.addParam('showFscCutoff', FloatParam, label='Display FSC cutoff', default=0.143) @@ -113,26 +112,42 @@ def _getVisualizeDict(self): # --------------------------- UTILS functions ------------------------------ - def _getIterationParametersFilename(self, iteration: int): - return self.protocol._getIterationParametersFilename(iteration) - + def _getIteration(self) -> int: + iteration = int(self.iteration) + if iteration < 0: + return self._getCompletedIterationCount() + iteration + else: + return iteration + + def _getClass(self) -> int: + return int(self.classId) - 1 + def _getIterationCount(self) -> int: return self.protocol._getIterationCount() + def _getCompletedIterationCount(self) -> int: + return self.protocol._getCompletedIterationCount() + def _getClassCount(self) -> int: return self.protocol._getClassCount() def _getSamplingRate(self) -> float: return self.protocol._getSamplingRate() + def _getIterationPath(self, iteration: int, *paths) -> str: + return self.protocol._getIterationPath(iteration *paths) + + def _getIterationParametersFilename(self, iteration: int): + return self.protocol._getIterationParametersFilename(iteration) + def _getAlignmentMdFilename(self, iteration: int): return self.protocol._getAlignmentMdFilename(iteration) def _getFscFilename(self, iteration: int, cls: int): return self.protocol._getFscFilename(iteration, cls) - def _getFilteredReconstructionFilename(self, iteration: int, cls: int): - return self.protocol._getFilteredReconstructionFilename(iteration, cls) + def _getFilteredVolumeFilename(self, iteration: int, cls: int): + return self.protocol._getFilteredVolumeFilename(iteration, cls) def _getAngleDiffMdFilename(self, iteration: int): return self.protocol._getAngleDiffMdFilename(iteration) @@ -156,7 +171,7 @@ def _readFsc(self, fscMd: emlib.MetaData) -> np.ndarray: fsc = fscMd.getValue(emlib.MDL_RESOLUTION_FRC, objId) values.append((freq, fsc)) - return np.array(values) + return np.array(values) def _iterFilenamesUntilNotExist(self, filenames: Iterable[str]): for filename in filenames: @@ -345,7 +360,8 @@ def _showNoiseModel(self, e): ax.set_title('Noise model') ax.set_xlabel('Resolution (1/A)') ax.set_ylabel('$\sigma^2$') - ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) + ax.set_xlim((0.0, 0.5)) + ax.set_xticks(ax.get_xticks(), self._computeFscTickLabels(ax.get_xticks())) ax.legend() return [fig] @@ -365,7 +381,8 @@ def _showWeights(self, e): ax.set_title('Weights') ax.set_xlabel('Resolution (1/A)') ax.set_ylabel('Weight') - ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) + ax.set_xlim((0.0, 0.5)) + ax.set_xticks(ax.get_xticks(), self._computeFscTickLabels(ax.get_xticks())) ax.legend() return [fig] @@ -404,7 +421,8 @@ def _showClassSizes(self, e): return [fig] def _showAngleDiffMetadata(self, e): - mdFilename = self._getAngleDiffMdFilename(int(self.showAngleDiffMetadata)) + iteration = self._getIteration() + mdFilename = self._getAngleDiffMdFilename(iteration) v = DataView(mdFilename) return [v] @@ -439,12 +457,14 @@ def _showAngleDiffShiftDiffHistogram(self, e): return [fig] def _showIterationVolume(self, e): - iteration = int(self.showIterationVolume) - filename = self._getFilteredReconstructionFilename(iteration, 0) - return DataView(filename) + iteration = self._getIteration() + cls = self._getClass() + filename = self._getFilteredVolumeFilename(iteration, cls) + view = DataView(filename) + return [view] def _showIterationAngularDistribution(self, e): - iteration = int(self.showIterationAngularDistribution) + iteration = self._getIteration() view = EmPlotter(x=1, y=1, mainTitle="Iteration %d" % iteration, windowTitle="Angular distribution") axis = view.plotAngularDistributionFromMd(self._getAlignmentMdFilename(iteration), '', histogram=True) view.getFigure().colorbar(axis) @@ -455,7 +475,7 @@ def _showIterationAngularDistribution3d(self, e): ax = fig.add_subplot(111, projection='3d') # Read data - iteration = int(self.showIterationAngularDistribution3d) + iteration = self._getIteration() md = emlib.MetaData(self._getAlignmentMdFilename(iteration)) rot = np.deg2rad(md.getColumnValues(emlib.MDL_ANGLE_ROT)) tilt = np.deg2rad(md.getColumnValues(emlib.MDL_ANGLE_TILT)) @@ -485,7 +505,7 @@ def _showIterationAngularDistribution3d(self, e): def _showIterationShiftHistogram(self, e): fig, ax = plt.subplots() - iteration = int(self.showIterationShiftHistogram) + iteration = self._getIteration() md = emlib.MetaData(self._getAlignmentMdFilename(iteration)) shiftX = md.getColumnValues(emlib.MDL_SHIFT_X) @@ -510,8 +530,8 @@ def _showIterationShiftHistogram(self, e): def _showIterationFsc(self, e): fig, ax = plt.subplots() - it = int(self.showIterationFsc) - for cls, fscFn in enumerate(self._iterFscMdIterationFilenames(it), start=1): + iteration = self._getIteration() + for cls, fscFn in enumerate(self._iterFscMdIterationFilenames(iteration), start=1): fscMd = emlib.MetaData(fscFn) fsc = self._readFsc(fscMd) label = f'Class {cls}' @@ -519,10 +539,11 @@ def _showIterationFsc(self, e): ax.axhline(0.5, color='black', linestyle='--') ax.axhline(0.143, color='black', linestyle='--') - ax.set_title('Class FSC') + ax.set_title('Iteration %d FSC' % iteration) ax.set_xlabel('Resolution (1/A)') ax.set_ylabel('FSC') - ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) + ax.set_xlim((0.0, 0.5)) + ax.set_xticks(ax.get_xticks(), self._computeFscTickLabels(ax.get_xticks())) ax.legend() return [fig] @@ -530,7 +551,7 @@ def _showIterationFsc(self, e): def _showClassFsc(self, e): fig, ax = plt.subplots() - cls = int(self.showClassFsc) + cls = self._getClass() for it, fscFn in enumerate(self._iterFscMdClassFilenames(cls), start=1): fscMd = emlib.MetaData(fscFn) fsc = self._readFsc(fscMd) @@ -539,10 +560,11 @@ def _showClassFsc(self, e): ax.axhline(0.5, color='black', linestyle='--') ax.axhline(0.143, color='black', linestyle='--') - ax.set_title('Class FSC') + ax.set_title('Class %d FSC' % (cls+1)) ax.set_xlabel('Resolution (1/A)') ax.set_ylabel('FSC') - ax.set_xticklabels(self._computeFscTickLabels(ax.get_xticks())) + ax.set_xlim((0.0, 0.5)) + ax.set_xticks(ax.get_xticks(), self._computeFscTickLabels(ax.get_xticks())) ax.legend() return [fig] From 1379e931456c2acf641c213b41aba067355d29e5 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 23 Mar 2023 16:22:08 +0100 Subject: [PATCH 063/104] Added psi visualization --- .../protocol_reconstruct_swiftres.py | 1 - xmipp3/viewers/viewer_reconstruct_swiftres.py | 25 +++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 96398f01e..11e030b1f 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -902,7 +902,6 @@ def _computeAlignmentConsensus( self, row.setValue(emlib.MDL_SHIFT_Y+1, mds[0].getValue(emlib.MDL_SHIFT_Y, objIds[0])) row.setValue(emlib.MDL_SHIFT_Y+2, mds[1].getValue(emlib.MDL_SHIFT_Y, objIds[1])) - row.writeToMd(resultMd, resultMd.addObject()) return resultMd diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index e49face90..0201d3baf 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -472,21 +472,26 @@ def _showIterationAngularDistribution(self, e): def _showIterationAngularDistribution3d(self, e): fig = plt.figure() - ax = fig.add_subplot(111, projection='3d') # Read data iteration = self._getIteration() md = emlib.MetaData(self._getAlignmentMdFilename(iteration)) rot = np.deg2rad(md.getColumnValues(emlib.MDL_ANGLE_ROT)) tilt = np.deg2rad(md.getColumnValues(emlib.MDL_ANGLE_TILT)) + psi = np.deg2rad(md.getColumnValues(emlib.MDL_ANGLE_PSI)) # Make histogram N = 32 heatmap, rotEdges, tiltEdges = np.histogram2d( x=rot, y=tilt, + range=[(-np.pi, np.pi), (0.0, np.pi)], + density=True, + bins=N + ) + hist, psiEdges = np.histogram( + psi, density=True, - range=[[-np.pi, +np.pi], [0.0, np.pi]], bins=N ) @@ -496,9 +501,15 @@ def _showIterationAngularDistribution3d(self, e): y = np.sin(tiltEdges) * np.sin(rotEdges) z = np.cos(tiltEdges) - # Plot surface - ax.plot_surface(x, y, z, rstride=1, cstride=1, facecolors=cm.plasma(heatmap)) - ax.set_title('Angular distribution') + # Plot the sphere + ax1 = fig.add_subplot(1, 2, 1, projection='3d') + ax1.plot_surface(x, y, z, rstride=1, cstride=1, facecolors=cm.plasma(heatmap)) + ax1.set_title('TILT and ROT distribution histogram') + + # Plot the + ax2 = fig.add_subplot(1, 2, 2, projection='polar') + ax2.bar((psiEdges[:-1] + psiEdges[1:])/2, hist) + ax2.set_title('PSI distribution histogram') return [fig] @@ -534,7 +545,7 @@ def _showIterationFsc(self, e): for cls, fscFn in enumerate(self._iterFscMdIterationFilenames(iteration), start=1): fscMd = emlib.MetaData(fscFn) fsc = self._readFsc(fscMd) - label = f'Class {cls}' + label = f'Class {cls+1}' ax.plot(fsc[:,0], fsc[:,1], label=label) ax.axhline(0.5, color='black', linestyle='--') ax.axhline(0.143, color='black', linestyle='--') @@ -576,7 +587,7 @@ def _showFscCutoff(self, e): for cls in range(self._getClassCount()): y = self._readFscCutoff(cls, cutoff) x = np.arange(len(y)) - label = f'Class {cls}' + label = f'Class {cls+1}' ax.plot(x, y, label=label) ax.set_title('Resolution') From f11d0d0f06c29d1fd08f5a93135fa3fc56abe359 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 28 Mar 2023 19:02:03 +0200 Subject: [PATCH 064/104] Added fp16 support --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 11e030b1f..2a9d7d6de 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -98,9 +98,13 @@ def _defineParams(self, form: Form): form.addSection(label='Compute') form.addParam('databaseRecipe', StringParam, label='Database recipe', - default='OPQ48_192,IVF32768,PQ48', expertLevel=LEVEL_ADVANCED, + default='OPQ48_192,IVF32768,PQ48', help='FAISS database structure. Please refer to ' 'https://github.com/facebookresearch/faiss/wiki/The-index-factory') + form.addParam('useFloat16', BooleanParam, label='Use float16', default=False, + help='When enabled, FAISS will be prompted to use half precision floating point ' + 'numbers. This may improve performance and/or memory footprint at some ' + 'accuracy cost. Only supported for GPUs') form.addParam('databaseTrainingSetSize', IntParam, label='Database training set size', default=int(2e6), help='Number of data-augmented particles to used when training the database') @@ -386,6 +390,8 @@ def trainDatabaseStep(self, iteration: int): args += ['--scratch', self._getTrainingScratchFilename()] if self.useGpu: args += ['--device', 'cuda:0'] # TODO select + if self.useFloat16: + args += ['--fp16'] env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it From 0ff049bba7b027eace64a12fe6b902ac11652b2f Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 3 Apr 2023 08:45:43 +0200 Subject: [PATCH 065/104] Added mask parameter --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 2a9d7d6de..3ebcd0f08 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -76,6 +76,8 @@ def _defineParams(self, form: Form): form.addParam('symmetryGroup', StringParam, default='c1', label='Symmetry group', help='If no symmetry is present, give c1') + form.addParam('mask', PointerParam, label="Mask", pointerClass='VolumeMask', allowsNull=True, + help='The mask values must be between 0 (remove these pixels) and 1 (let them pass). Smooth masks are recommended.') form.addSection(label='Global refinement') form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) From c8bba25e44fdb6411ec8dd9ab2fea44a3463e163 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 3 Apr 2023 08:45:54 +0200 Subject: [PATCH 066/104] Added Chimera viewer --- xmipp3/viewers/viewer_reconstruct_swiftres.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index 0201d3baf..7a6e9a4b7 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -38,7 +38,7 @@ from pwem import emlib from pwem.viewers.views import DataView, ObjectView -from pwem.viewers import EmPlotter +from pwem.viewers import EmPlotter, ChimeraAngDist import xmippLib @@ -77,6 +77,7 @@ def _defineParams(self, form): form.addSection(label='Reconstruction') form.addParam('showIterationAngularDistribution', LabelParam, label='Display iteration angular distribution') form.addParam('showIterationAngularDistribution3d', LabelParam, label='Display iteration angular distribution 3D') + form.addParam('showIterationAngularDistributionChimera', LabelParam, label='Display iteration angular distribution in Chimera') form.addParam('showIterationShiftHistogram', LabelParam, label='Display iteration shift histogram') form.addParam('showIterationVolume', LabelParam, label='Display iteration volume') @@ -101,6 +102,7 @@ def _getVisualizeDict(self): 'showIterationAngularDistribution': self._showIterationAngularDistribution, 'showIterationAngularDistribution3d': self._showIterationAngularDistribution3d, + 'showIterationAngularDistributionChimera': self._showIterationAngularDistributionChimera, 'showIterationShiftHistogram': self._showIterationShiftHistogram, 'showIterationVolume': self._showIterationVolume, @@ -513,6 +515,23 @@ def _showIterationAngularDistribution3d(self, e): return [fig] + def _showIterationAngularDistributionChimera(self, e): + iteration = self._getIteration() + cls = self._getClass() + + volFn = self._getFilteredVolumeFilename(iteration, cls) + alignmentMdFn = self._getAlignmentMdFilename(iteration) + volOrigin = (0, 0, 0) #TODO + + view = ChimeraAngDist( + volFile=volFn, + tmpFilesPath=self.protocol._getPath(), + angularDistFile=alignmentMdFn, + volOrigin=volOrigin + ) + + return [view] + def _showIterationShiftHistogram(self, e): fig, ax = plt.subplots() From 27c88ba7651b5b6c4613457f8ca9014fc17594f1 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 12 Apr 2023 12:02:46 +0200 Subject: [PATCH 067/104] Added knn support --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 3ebcd0f08..c107bb3dd 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -97,6 +97,8 @@ def _defineParams(self, form: Form): help='Maximum shift of the particle in pixels') form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, help='Percentage of best particles used for reconstruction') + form.addParam('numberOfMatches', IntParam, label='Number of matches', default=1, + help='Number of reference matches for each particle') form.addSection(label='Compute') form.addParam('databaseRecipe', StringParam, label='Database recipe', @@ -425,6 +427,7 @@ def alignStep(self, iteration: int, repetition: int, local: int): args += ['--dropna'] args += ['--batch', self.batchSize] args += ['--max_size', self.databaseMaximumSize] + args += ['-k', self.numberOfMatches] if self.useGpu: args += ['--device', 'cuda:0'] # TODO select if local > 0: From 537a2f1e7b0050cb597f2564a6f1f88d6b4cfcb0 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 13 Apr 2023 08:56:35 +0200 Subject: [PATCH 068/104] Renamed conda env --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index c107bb3dd..4a9c8deba 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -46,7 +46,7 @@ class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): _label = 'swiftres' - _conda_env = 'xmipp_torch' + _conda_env = 'xmipp_swiftres' def __init__(self, **kwargs): ProtRefine3D.__init__(self, **kwargs) From ea1505eca7a5712b8347781d5440ab1252dd59f2 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 13 Apr 2023 11:56:43 +0200 Subject: [PATCH 069/104] Renamed environment and programs to swiftalign --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 4a9c8deba..2f47e2bb0 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -46,7 +46,7 @@ class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): _label = 'swiftres' - _conda_env = 'xmipp_swiftres' + _conda_env = 'xmipp_swiftalign' def __init__(self, **kwargs): ProtRefine3D.__init__(self, **kwargs) @@ -399,7 +399,7 @@ def trainDatabaseStep(self, iteration: int): env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it - self.runJob('xmipp_train_database', args, numberOfMpi=1, env=env) + self.runJob('xmipp_swiftalign_train', args, numberOfMpi=1, env=env) def alignStep(self, iteration: int, repetition: int, local: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) @@ -435,7 +435,7 @@ def alignStep(self, iteration: int, repetition: int, local: int): env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it - self.runJob('xmipp_query_database', args, numberOfMpi=1, env=env) + self.runJob('xmipp_swiftalign_query', args, numberOfMpi=1, env=env) def alignmentConsensusStep(self, iteration: int): paramMd = emlib.MetaData(self._getIterationParametersFilename(iteration)) From 7a7fb81e5dc6a0bb0819d9fedd426270c358d9e8 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 21 Apr 2023 12:29:40 +0200 Subject: [PATCH 070/104] Commented unused steps --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 2f47e2bb0..2bad3aea0 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -165,7 +165,7 @@ def _insertProjectSteps(self, iteration: int, prerequisites): # Merge galleries of classes mergeStepIds.append(self._insertFunctionStep('mergeGalleriesStep', iteration, repetition, prerequisites=projectStepIds)) - mergeStepIds.append(self._insertFunctionStep('computeNoiseModelStep', iteration, prerequisites=prerequisites)) + #mergeStepIds.append(self._insertFunctionStep('computeNoiseModelStep', iteration, prerequisites=prerequisites)) return mergeStepIds @@ -184,7 +184,7 @@ def _insertAlignmentSteps(self, iteration: int, local: int, prerequisites): def _insertReconstructSteps(self, iteration: int, cls: int, prerequisites): selectAlignmentStepId = self._insertFunctionStep('selectAlignmentStep', iteration, cls, prerequisites=prerequisites) - compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[selectAlignmentStepId]) + #compareReprojectionStepId = self._insertFunctionStep('compareReprojectionStep', iteration, cls, prerequisites=[selectAlignmentStepId]) #computeWeightsStepId = self._insertFunctionStep('computeWeightsStep', iteration, cls, prerequisites=[compareReprojectionStepId]) #filterByWeightsStepId = self._insertFunctionStep('filterByWeightsStep', iteration, cls, prerequisites=[computeWeightsStepId]) splitStepId = self._insertFunctionStep('splitStep', iteration, cls, prerequisites=[selectAlignmentStepId]) From fd0e7c8125ac9867110ac150662502a1c47a7961 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 21 Apr 2023 13:29:37 +0200 Subject: [PATCH 071/104] Made CTF optional --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 2bad3aea0..25881ff2a 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -79,6 +79,11 @@ def _defineParams(self, form: Form): form.addParam('mask', PointerParam, label="Mask", pointerClass='VolumeMask', allowsNull=True, help='The mask values must be between 0 (remove these pixels) and 1 (let them pass). Smooth masks are recommended.') + form.addSection(label='CTF') + form.addParam('considerInputCtf', BooleanParam, label='Consider CTF', + default=True, + help='Consider the CTF of the particles') + form.addSection(label='Global refinement') form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) form.addParam('numberOfAlignmentRepetitions', IntParam, label='Number of repetitions', default=2) @@ -123,7 +128,14 @@ def _defineParams(self, form: Form): form.addParallelSection(threads=1, mpi=8) #--------------------------- INFO functions -------------------------------------------- + def _validate(self): + errors = [] + particles = self.inputParticles.get() + + if self.considerInputCtf and not particles.hasCTF(): + errors.append('Input must have CTF information for being able to consider it') + return errors #--------------------------- INSERT steps functions -------------------------------------------- def _insertAllSteps(self): @@ -207,7 +219,7 @@ def convertInputStep(self): def correctCtfStep(self): particles = self.inputParticles.get() - if particles.hasCTF(): + if self.considerInputCtf: args = [] args += ['-i', self._getInputParticleMdFilename()] args += ['-o', self._getWienerParticleStackFilename()] From 0c2978bea57af8555defacc9a9ebba21593b6a18 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 24 Apr 2023 09:08:43 +0200 Subject: [PATCH 072/104] Added manual step options --- .../protocols/protocol_reconstruct_swiftres.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 25881ff2a..d8eecb454 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -76,6 +76,8 @@ def _defineParams(self, form: Form): form.addParam('symmetryGroup', StringParam, default='c1', label='Symmetry group', help='If no symmetry is present, give c1') + form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0, + help='Image comparison resolution limit at the first iteration of the refinement') form.addParam('mask', PointerParam, label="Mask", pointerClass='VolumeMask', allowsNull=True, help='The mask values must be between 0 (remove these pixels) and 1 (let them pass). Smooth masks are recommended.') @@ -87,9 +89,7 @@ def _defineParams(self, form: Form): form.addSection(label='Global refinement') form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) form.addParam('numberOfAlignmentRepetitions', IntParam, label='Number of repetitions', default=2) - form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0, - help='Image comparison resolution limit at the first iteration of the refinement') - form.addParam('maximumResolution', FloatParam, label="Maximum resolution (A)", default=8.0, + form.addParam('maximumResolution', FloatParam, label="Maximum alignment resolution (A)", default=8.0, help='Image comparison resolution limit of the refinement') form.addParam('nextResolutionCriterion', FloatParam, label="FSC criterion", default=0.5, expertLevel=LEVEL_ADVANCED, @@ -100,6 +100,12 @@ def _defineParams(self, form: Form): help='Maximum psi parameter of the particles') form.addParam('initialMaxShift', FloatParam, label='Maximum shift (px)', default=16.0, help='Maximum shift of the particle in pixels') + form.addParam('useAutomaticStep', BooleanParam, label='Use automatic step', default=True, + expertLevel=LEVEL_ADVANCED, + help='Automatically determine the step used when exploring the projection landscape') + stepGroup = form.addGroup('Steps', condition='not useAutomaticStep', expertLevel=LEVEL_ADVANCED) + stepGroup.addParam('angleStep', FloatParam, label='Angle step (deg)', default=5.0) + stepGroup.addParam('shiftStep', FloatParam, label='Shift step (px)', default=2.0) form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, help='Percentage of best particles used for reconstruction') form.addParam('numberOfMatches', IntParam, label='Number of matches', default=1, @@ -283,8 +289,8 @@ def setupIterationStep(self, iteration: int, local: int): frequency = self._getSamplingRate() / resolution maxPsi = self._getIterationMaxPsi(iteration) maxShift = self._getIterationMaxShift(iteration) - shiftStep = self._computeShiftStep(frequency) - angleStep = self._computeAngleStep(frequency, imageSize) + shiftStep = self._computeShiftStep(frequency) if self.useAutomaticStep else float(self.shiftStep) + angleStep = self._computeAngleStep(frequency, imageSize) if self.useAutomaticStep else float(self.angleStep) maxResolution = max(resolution, float(self.maximumResolution)) maxFrequency = self._getSamplingRate() / maxResolution From c217a1decf3b8aa6f080cb807933ff057dc0a99a Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 8 May 2023 08:18:46 +0200 Subject: [PATCH 073/104] Added CTF grouping --- .../protocol_reconstruct_swiftres.py | 132 +++++++++++------- 1 file changed, 82 insertions(+), 50 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index d8eecb454..fffc0a157 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -21,7 +21,7 @@ # ***************************************************************************/ from pwem.protocols import ProtRefine3D -from pwem.objects import Volume, FSC, SetOfVolumes, Class3D +from pwem.objects import Volume, FSC, SetOfVolumes, Class3D, SetOfParticles from pwem.convert import transformations from pwem import emlib @@ -43,6 +43,7 @@ from scipy import stats import itertools import collections +import os class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): _label = 'swiftres' @@ -146,9 +147,13 @@ def _validate(self): #--------------------------- INSERT steps functions -------------------------------------------- def _insertAllSteps(self): convertInputStepId = self._insertFunctionStep('convertInputStep', prerequisites=[]) - correctCtfStepId = self._insertFunctionStep('correctCtfStep', prerequisites=[convertInputStepId]) - lastIds = [correctCtfStepId] + if self.considerInputCtf: + correctCtfStepId = self._insertFunctionStep('correctCtfStep', prerequisites=[convertInputStepId]) + lastIds = [correctCtfStepId] + else: + lastIds = [convertInputStepId] + for i in range(self._getIterationCount()): lastIds = self._insertIterationSteps(i, 0, prerequisites=lastIds) @@ -156,10 +161,11 @@ def _insertAllSteps(self): def _insertIterationSteps(self, iteration: int, local: int, prerequisites): setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, local, prerequisites=prerequisites) - #ctfGroupStepId = self._insertFunctionStep('ctfGroupStep', iteration, prerequisites=[setupIterationStepId]) + ctfGroupStepIds = [] + if self.considerInputCtf: + ctfGroupStepIds.append(self._insertFunctionStep('ctfGroupStep', iteration, prerequisites=[setupIterationStepId])) projectIds = self._insertProjectSteps(iteration, prerequisites=[setupIterationStepId]) - #alignIds = self._insertAlignmentSteps(iteration, local, prerequisites=projectIds + [ctfGroupStepId]) - alignIds = self._insertAlignmentSteps(iteration, local, prerequisites=projectIds) + alignIds = self._insertAlignmentSteps(iteration, local, prerequisites=projectIds + ctfGroupStepIds) compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, prerequisites=alignIds) ids = [] @@ -219,36 +225,48 @@ def _insertPostProcessSteps(self, iteration: int, cls: int, prerequisites): #--------------------------- STEPS functions -------------------------------------------- def convertInputStep(self): - writeSetOfParticles(self.inputParticles.get(), - self._getInputParticleMdFilename()) - - def correctCtfStep(self): - particles = self.inputParticles.get() + particles: SetOfParticles = self.inputParticles.get() - if self.considerInputCtf: - args = [] - args += ['-i', self._getInputParticleMdFilename()] - args += ['-o', self._getWienerParticleStackFilename()] - args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] - args += ['--keep_input_columns'] - args += ['--sampling_rate', self._getSamplingRate()] - args += ['--pad', '2'] - args += ['--wc', '-1.0'] - if particles.isPhaseFlipped(): - args += ['--phase_flipped'] + writeSetOfParticles(particles, + self._getInputParticleMdFilename()) - self.runJob('xmipp_ctf_correct_wiener2d', args) - - else: - # TODO When the stack is already in MRC format, simply link + def is_mrc(path: str) -> bool: + _, ext = os.path.splitext(path) + return ext == 'mrc' or ext == 'mrcs' + + # Convert to MRC if necessary + if not all(map(is_mrc, particles.getFiles())): args = [] args += ['-i', self._getInputParticleMdFilename()] - args += ['-o', self._getWienerParticleStackFilename()] - args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] + args += ['-o', self._getInputParticleStackFilename()] + args += ['--save_metadata_stack', self._getInputParticleMdFilename()] args += ['--keep_input_columns'] self.runJob('xmipp_image_convert', args, numberOfMpi=1) - + + def correctCtfStep(self): + particles = self.inputParticles.get() + + # Perform a CTF correction using Wiener Filtering + args = [] + args += ['-i', self._getInputParticleMdFilename()] + args += ['-o', self._getWienerParticleStackFilename()] + args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] + args += ['--sampling_rate', self._getSamplingRate()] + args += ['--pad', '2'] + args += ['--wc', -1] + if particles.isPhaseFlipped(): + args += ['--phase_flipped'] + + self.runJob('xmipp_ctf_correct_wiener2d', args) + + # Append the wiener corrected images to the second image label of the input + inputMd = emlib.MetaData(self._getInputParticleMdFilename()) + wienerMd = emlib.MetaData(self._getWienerParticleMdFilename()) + wienerImageFns = wienerMd.getColumnValues(emlib.MDL_IMAGE) + inputMd.setColumnValues(emlib.MDL_IMAGE1, wienerImageFns) + inputMd.write(self._getInputParticleMdFilename()) + def setupIterationStep(self, iteration: int, local: int): makePath(self._getIterationPath(iteration)) @@ -260,7 +278,7 @@ def setupIterationStep(self, iteration: int, local: int): # The input particles. In order to use union, columns must # match, so join columns prior to doing the union args = [] - args += ['-i', self._getWienerParticleMdFilename()] + args += ['-i', self._getInputParticleMdFilename()] args += ['-o', self._getIterationInputParticleMdFilename(iteration)] args += ['--set', 'join', self._getAlignmentMdFilename(iteration-1), 'itemId'] self._runMdUtils(args) @@ -276,7 +294,7 @@ def setupIterationStep(self, iteration: int, local: int): else: # For the first iteration, simply use the input particles. createLink( - self._getWienerParticleMdFilename(), + self._getInputParticleMdFilename(), self._getIterationInputParticleMdFilename(iteration) ) @@ -314,21 +332,27 @@ def ctfGroupStep(self, iteration: int): args = [] args += ['--ctfdat', self._getInputParticleMdFilename()] - args += ['-o', oroot] + args += ['-o', oroot + ':mrc'] args += ['--sampling_rate', self._getSamplingRate()] - args += ['--pad', 2] - args += ['--error', 1.0] # TODO make param + args += ['--pad', 1] + args += ['--error', 0.5] # TODO make param args += ['--resol', resolution] if particles.isPhaseFlipped(): args += ['--phase_flipped'] self.runJob('xmipp_ctf_group', args, numberOfMpi=1) + # Convert group info + groups = emlib.MetaData('groups@' + oroot + 'Info.xmd') + representatives = [str(i+1) + '@' + self._getCtfGroupAveragesFilename(iteration) for i in range(groups.size())] + groups.setColumnValues(emlib.MDL_IMAGE, representatives) + groups.write(self._getCtfGroupInfoMdFilename(iteration)) + # Rename files and removed unused ones cleanPath(oroot + 'Info.xmd') cleanPath(oroot + '_split.doc') moveFile(oroot + '_images.sel', self._getCtfGroupMdFilename(iteration)) - moveFile(oroot + '_ctf.mrcs', self._getCtfGroupAveragesFilename(iteration)) + moveFile(oroot + '_ctf.mrc', self._getCtfGroupAveragesFilename(iteration)) def projectVolumeStep(self, iteration: int, cls: int, repetition: int): md = emlib.MetaData(self._getIterationParametersFilename(iteration)) @@ -414,6 +438,9 @@ def trainDatabaseStep(self, iteration: int): args += ['--device', 'cuda:0'] # TODO select if self.useFloat16: args += ['--fp16'] + if self.considerInputCtf: + args += ['--ctf', self._getCtfGroupInfoMdFilename(iteration)] + env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it @@ -450,6 +477,8 @@ def alignStep(self, iteration: int, repetition: int, local: int): args += ['--device', 'cuda:0'] # TODO select if local > 0: args += ['--local_shift', '--local_psi'] + if self.considerInputCtf: + args += ['--ctf', self._getCtfGroupInfoMdFilename(iteration)] env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it @@ -533,7 +562,6 @@ def compareReprojectionStep(self, iteration: int, cls: int): args = [] args += ['-i', self._getReconstructionMdFilename(iteration, cls)] args += ['--ref', self._getIterationInputVolumeFilename(iteration, cls)] - args += ['--ignoreCTF'] # As we're using wiener corrected images args += ['--doNotWriteStack'] # Do not undo shifts self.runJob('xmipp_angular_continuous_assign2', args, numberOfMpi=self.numberOfMpi.get()) @@ -562,6 +590,12 @@ def filterByWeightsStep(self, iteration: int, cls: int): self._runMdUtils(args) def splitStep(self, iteration: int, cls: int): + if self.considerInputCtf: + args = [] + args += ['-i', self._getReconstructionMdFilename(iteration, cls)] + args += ['--operate', 'modify_values', 'image=image1'] + self._runMdUtils(args) + args = [] args += ['-i', self._getReconstructionMdFilename(iteration, cls)] args += ['-n', 2] @@ -657,20 +691,12 @@ def mergeAlignmentsStep(self, iteration: int): def createOutputStep(self): lastIteration = self._getIterationCount() - 1 - # Rename the wiener filtered image column - args = [] - args += ['-i', self._getAlignmentMdFilename(lastIteration)] - args += ['-o', self._getOutputParticlesMdFilename()] - args += ['--operate', 'rename_column', 'image image1'] - self._runMdUtils(args) - - # Add the input image column - args = [] - args += ['-i', self._getOutputParticlesMdFilename()] - args += ['--set', 'join', self._getInputParticleMdFilename(), 'itemId'] - self._runMdUtils(args) - # Link last iteration + createLink( + self._getAlignmentMdFilename(lastIteration), + self._getOutputParticlesMdFilename() + ) + for cls in range(self._getClassCount()): for i in range(1, 3): createLink( @@ -724,6 +750,9 @@ def _getClassPath(self, iteration: int, cls: int, *paths): def _getInputParticleMdFilename(self): return self._getExtraPath('input_particles.xmd') + def _getInputParticleStackFilename(self): + return self._getExtraPath('input_particles.mrcs') + def _getWienerParticleMdFilename(self): return self._getExtraPath('input_particles_wiener.xmd') @@ -748,11 +777,14 @@ def _getIterationInputParticleMdFilename(self, iteration: int): def _getCtfGroupOutputRoot(self, iteration: int): return self._getIterationPath(iteration, 'ctf_group') + def _getCtfGroupInfoMdFilename(self, iteration: int): + return self._getIterationPath(iteration, 'ctf_group_info.xmd') + def _getCtfGroupMdFilename(self, iteration: int): return self._getIterationPath(iteration, 'ctf_groups.xmd') def _getCtfGroupAveragesFilename(self, iteration: int): - return self._getIterationPath(iteration, 'ctfs.mrcs') + return self._getIterationPath(iteration, 'ctf_group_representatives.mrcs') def _getClassGalleryMdFilename(self, iteration: int, cls: int, repetition: int): return self._getClassPath(iteration, cls, 'gallery%05d.doc' % repetition) From e5a49b268f6a5c5574d19a2d24958c1c93929230 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 8 May 2023 08:31:22 +0200 Subject: [PATCH 074/104] Fixed mrc detection --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index fffc0a157..b362e3704 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -232,7 +232,7 @@ def convertInputStep(self): def is_mrc(path: str) -> bool: _, ext = os.path.splitext(path) - return ext == 'mrc' or ext == 'mrcs' + return ext == '.mrc' or ext == '.mrcs' # Convert to MRC if necessary if not all(map(is_mrc, particles.getFiles())): From 16bd8c86e82908ca939b35fbe46a061592ce2cc7 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 10 May 2023 11:06:11 +0200 Subject: [PATCH 075/104] Added option to copy particles to scratchdir --- .../protocol_reconstruct_swiftres.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index b362e3704..7baf20ffa 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -131,6 +131,9 @@ def _defineParams(self, form: Form): form.addParam('batchSize', IntParam, label='Batch size', default=8192, help='It is recommended to use powers of 2. Using numbers around 8192 works well') + form.addParam('copyParticles', BooleanParam, label='Copy particles to scratch', default=False, + help='Copy input particles to scratch directory. Note that if input file format is ' + 'incompatible the particles will be converted into scratch anyway') form.addParallelSection(threads=1, mpi=8) @@ -235,12 +238,13 @@ def is_mrc(path: str) -> bool: return ext == '.mrc' or ext == '.mrcs' # Convert to MRC if necessary - if not all(map(is_mrc, particles.getFiles())): + if self.copyParticles or not all(map(is_mrc, particles.getFiles())): args = [] args += ['-i', self._getInputParticleMdFilename()] args += ['-o', self._getInputParticleStackFilename()] args += ['--save_metadata_stack', self._getInputParticleMdFilename()] args += ['--keep_input_columns'] + args += ['--track_origin'] self.runJob('xmipp_image_convert', args, numberOfMpi=1) @@ -692,10 +696,18 @@ def createOutputStep(self): lastIteration = self._getIterationCount() - 1 # Link last iteration - createLink( - self._getAlignmentMdFilename(lastIteration), - self._getOutputParticlesMdFilename() - ) + if os.path.exists(self._getInputParticleStackFilename()): + # Use original images + alignmentMd = emlib.MetaData(self._getAlignmentMdFilename(lastIteration)) + alignmentMd.copyColumn(emlib.MDL_IMAGE, emlib.MDL_IMAGE_ORIGINAL) + alignmentMd.write(self._getOutputParticlesMdFilename()) + + else: + # Link + createLink( + self._getAlignmentMdFilename(lastIteration), + self._getOutputParticlesMdFilename() + ) for cls in range(self._getClassCount()): for i in range(1, 3): @@ -751,7 +763,7 @@ def _getInputParticleMdFilename(self): return self._getExtraPath('input_particles.xmd') def _getInputParticleStackFilename(self): - return self._getExtraPath('input_particles.mrcs') + return self._getTmpPath('input_particles.mrcs') def _getWienerParticleMdFilename(self): return self._getExtraPath('input_particles_wiener.xmd') From c79faa8c346fb88f4657d30849a6949d7a14f3df Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 26 May 2023 17:59:22 +0200 Subject: [PATCH 076/104] Refactored output generation --- .../protocol_reconstruct_swiftres.py | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 7baf20ffa..157e1c323 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -21,7 +21,7 @@ # ***************************************************************************/ from pwem.protocols import ProtRefine3D -from pwem.objects import Volume, FSC, SetOfVolumes, Class3D, SetOfParticles +from pwem.objects import Volume, FSC, SetOfVolumes, Class3D, SetOfParticles, SetOfClasses3D, Particle from pwem.convert import transformations from pwem import emlib @@ -34,7 +34,7 @@ createLink, cleanPattern) import xmipp3 -from xmipp3.convert import writeSetOfParticles, readSetOfParticles +from xmipp3.convert import writeSetOfParticles, rowToParticle from typing import Iterable, Sequence, Optional import math @@ -46,8 +46,15 @@ import os class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): + OUTPUT_CLASSES_NAME = 'classes' + OUTPUT_VOLUMES_NAME = 'volumes' + _label = 'swiftres' _conda_env = 'xmipp_swiftalign' + _possibleOutputs = { + OUTPUT_CLASSES_NAME: SetOfClasses3D, + OUTPUT_VOLUMES_NAME: SetOfVolumes + } def __init__(self, **kwargs): ProtRefine3D.__init__(self, **kwargs) @@ -309,7 +316,9 @@ def setupIterationStep(self, iteration: int, local: int): imageSize = self._getImageSize() frequency = self._getSamplingRate() / resolution + #maxPsi = self._getIterationMaxPsi(iteration) * math.pow(2, -local) maxPsi = self._getIterationMaxPsi(iteration) + #maxShift = self._getIterationMaxShift(iteration) * math.pow(2, -local) maxShift = self._getIterationMaxShift(iteration) shiftStep = self._computeShiftStep(frequency) if self.useAutomaticStep else float(self.shiftStep) angleStep = self._computeAngleStep(frequency, imageSize) if self.useAutomaticStep else float(self.angleStep) @@ -480,7 +489,7 @@ def alignStep(self, iteration: int, repetition: int, local: int): if self.useGpu: args += ['--device', 'cuda:0'] # TODO select if local > 0: - args += ['--local_shift', '--local_psi'] + args += ['--local'] if self.considerInputCtf: args += ['--ctf', self._getCtfGroupInfoMdFilename(iteration)] @@ -1009,9 +1018,8 @@ def _computeIterationResolution(self, iteration: int) -> float: def _computeAngleStep(self, maxFrequency: float, size: int) -> float: # At the alignment resolution limit, determine the # angle between to neighboring Fourier coefficients - c = maxFrequency*size # Cos: radius - s = 1.0 # Sin: 1 coefficient - angle = math.atan2(s, c) + s = 1.0 / (maxFrequency*size) + angle = math.asin(s) # The angular error is at most half of the sampling # Therefore use the double angular sampling @@ -1033,8 +1041,6 @@ def _getIterationMaxShift(self, iteration: int) -> float: return float(self.initialMaxShift) def _createOutputClasses3D(self, volumes: SetOfVolumes): - particles = self._createSetOfParticles() - EXTRA_LABELS = [ #emlib.MDL_COST, #emlib.MDL_WEIGHT, @@ -1044,25 +1050,28 @@ def _createOutputClasses3D(self, volumes: SetOfVolumes): #emlib.MDL_IMED ] - # Fill - readSetOfParticles( - self._getOutputParticlesMdFilename(), - particles, - extraLabels=EXTRA_LABELS - ) - particles.setSamplingRate(self._getSamplingRate()) - self._insertChild('outputParticles', particles) - + def updateItem(item: Particle, row: emlib.metadata.Row): + if row is not None: + particle: Particle = rowToParticle(row, extraLabels=EXTRA_LABELS) + item.copy(particle) + else: + item._appendItem = False + def updateClass(cls: Class3D): clsId = cls.getObjId() representative = volumes[clsId] cls.setRepresentative(representative) - classes3d = self._createSetOfClasses3D(particles) - classes3d.classifyItems(updateClassCallback=updateClass) + particlesMd = emlib.MetaData(self._getOutputParticlesMdFilename()) + classes3d = self._createSetOfClasses3D(self.inputParticles) + classes3d.classifyItems( + updateItemCallback=updateItem, + updateClassCallback=updateClass, + itemDataIterator=itertools.chain(emlib.metadata.iterRows(particlesMd), itertools.repeat(None)) + ) # Define the output - self._defineOutputs(outputClasses=classes3d) + self._defineOutputs(**{self.OUTPUT_CLASSES_NAME: classes3d}) self._defineSourceRelation(self.inputParticles, classes3d) self._defineSourceRelation(self.inputVolumes, classes3d) From aa98932239a276b49c9f89c8f993531b41b9612b Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 30 May 2023 10:15:40 +0200 Subject: [PATCH 077/104] Added local alignment option --- .../protocol_reconstruct_swiftres.py | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 157e1c323..a2b873cd6 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -96,6 +96,7 @@ def _defineParams(self, form: Form): form.addSection(label='Global refinement') form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) + form.addParam('numberOfLocalIterations', IntParam, label='Number of local iterations', default=4) form.addParam('numberOfAlignmentRepetitions', IntParam, label='Number of repetitions', default=2) form.addParam('maximumResolution', FloatParam, label="Maximum alignment resolution (A)", default=8.0, help='Image comparison resolution limit of the refinement') @@ -165,17 +166,18 @@ def _insertAllSteps(self): lastIds = [convertInputStepId] for i in range(self._getIterationCount()): - lastIds = self._insertIterationSteps(i, 0, prerequisites=lastIds) + lastIds = self._insertIterationSteps(i, prerequisites=lastIds) self._insertFunctionStep('createOutputStep', prerequisites=lastIds) - def _insertIterationSteps(self, iteration: int, local: int, prerequisites): - setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, local, prerequisites=prerequisites) + def _insertIterationSteps(self, iteration: int, prerequisites): + setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, prerequisites=prerequisites) ctfGroupStepIds = [] if self.considerInputCtf: ctfGroupStepIds.append(self._insertFunctionStep('ctfGroupStep', iteration, prerequisites=[setupIterationStepId])) + projectIds = self._insertProjectSteps(iteration, prerequisites=[setupIterationStepId]) - alignIds = self._insertAlignmentSteps(iteration, local, prerequisites=projectIds + ctfGroupStepIds) + alignIds = self._insertAlignmentSteps(iteration, prerequisites=projectIds + ctfGroupStepIds) compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, prerequisites=alignIds) ids = [] @@ -203,13 +205,16 @@ def _insertProjectSteps(self, iteration: int, prerequisites): return mergeStepIds - def _insertAlignmentSteps(self, iteration: int, local: int, prerequisites): + def _insertAlignmentSteps(self, iteration: int, prerequisites): ensembleTrainingSetStepId = self._insertFunctionStep('ensembleTrainingSetStep', iteration, prerequisites=prerequisites) trainDatabaseStepId = self._insertFunctionStep('trainDatabaseStep', iteration, prerequisites=[ensembleTrainingSetStepId]) alignStepIds = [] for repetition in range(self._getAlignmentRepetitionCount()): - alignStepIds.append(self._insertFunctionStep('alignStep', iteration, repetition, local, prerequisites=[trainDatabaseStepId])) + alignStepId = trainDatabaseStepId + for local in range(self._getLocalIterationCount()): + alignStepId = self._insertFunctionStep('alignStep', iteration, repetition, local, prerequisites=[alignStepId]) + alignStepIds.append(alignStepId) alignmentConsensusStepId = self._insertFunctionStep('alignmentConsensusStep', iteration, prerequisites=alignStepIds) intersectInputStepId = self._insertFunctionStep('intersectInputStep', iteration, prerequisites=[alignmentConsensusStepId]) @@ -278,7 +283,7 @@ def correctCtfStep(self): inputMd.setColumnValues(emlib.MDL_IMAGE1, wienerImageFns) inputMd.write(self._getInputParticleMdFilename()) - def setupIterationStep(self, iteration: int, local: int): + def setupIterationStep(self, iteration: int): makePath(self._getIterationPath(iteration)) for cls in range(self._getClassCount()): @@ -311,14 +316,9 @@ def setupIterationStep(self, iteration: int, local: int): resolution = float(self.initialResolution) - if local > 0: - raise NotImplementedError('Local searches are not implemented') - imageSize = self._getImageSize() frequency = self._getSamplingRate() / resolution - #maxPsi = self._getIterationMaxPsi(iteration) * math.pow(2, -local) maxPsi = self._getIterationMaxPsi(iteration) - #maxShift = self._getIterationMaxShift(iteration) * math.pow(2, -local) maxShift = self._getIterationMaxShift(iteration) shiftStep = self._computeShiftStep(frequency) if self.useAutomaticStep else float(self.shiftStep) angleStep = self._computeAngleStep(frequency, imageSize) if self.useAutomaticStep else float(self.angleStep) @@ -434,13 +434,13 @@ def trainDatabaseStep(self, iteration: int): maxPsi = md.getValue(emlib.MDL_ANGLE_PSI, 1) maxShiftPx = md.getValue(emlib.MDL_SHIFT_X, 1) maxShift = maxShiftPx / imageSize - + args = [] args += ['-i', self._getTrainingMdFilename(iteration)] args += ['-o', self._getTrainingIndexFilename(iteration)] args += ['--recipe', recipe] #args += ['--weights', self._getWeightsFilename(iteration)] - args += ['--max_shift', maxShift] + args += ['--max_shift', maxShift] #FIXME fails with != 190 args += ['--max_psi', maxPsi] args += ['--max_frequency', maxFrequency] args += ['--method', 'fourier'] @@ -468,10 +468,19 @@ def alignStep(self, iteration: int, repetition: int, local: int): maxShift = maxShiftPx / imageSize nShift = round((2*maxShiftPx) / md.getValue(emlib.MDL_SHIFT_DIFF, 1)) + 1 nRotations = round(360 / md.getValue(emlib.MDL_ANGLE_DIFF, 1)) - + + if local > 0: + inputMdFilename = self._getAlignmentRepetitionMdFilename(iteration, repetition) + else: + inputMdFilename = self._getIterationInputParticleMdFilename(iteration) + + localFactor = math.pow(2, -local) + maxPsi *= localFactor + maxShift /= localFactor + # Perform the alignment args = [] - args += ['-i', self._getIterationInputParticleMdFilename(iteration)] + args += ['-i', inputMdFilename] args += ['-o', self._getAlignmentRepetitionMdFilename(iteration, repetition)] args += ['-r', self._getGalleryMdFilename(iteration, repetition)] args += ['--index', self._getTrainingIndexFilename(iteration)] @@ -750,6 +759,9 @@ def _getCompletedIterationCount(self) -> int: def _getAlignmentRepetitionCount(self) -> int: return int(self.numberOfAlignmentRepetitions) + def _getLocalIterationCount(self) -> int: + return int(self.numberOfLocalIterations) + def _getClassCount(self) -> int: return len(self.inputVolumes) From 66329f80a9da4e74731852a87ee73f72500f242c Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 30 May 2023 15:52:52 +0200 Subject: [PATCH 078/104] Stupid mistake --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index a2b873cd6..ad065ff29 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -476,7 +476,7 @@ def alignStep(self, iteration: int, repetition: int, local: int): localFactor = math.pow(2, -local) maxPsi *= localFactor - maxShift /= localFactor + maxShift *= localFactor # Perform the alignment args = [] From ee424f3e394219ab0808406f4cea57471366d94e Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 31 May 2023 09:15:01 +0200 Subject: [PATCH 079/104] Adapted to the new alignment program --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index ad065ff29..3db80e1f1 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -495,6 +495,7 @@ def alignStep(self, iteration: int, repetition: int, local: int): args += ['--batch', self.batchSize] args += ['--max_size', self.databaseMaximumSize] args += ['-k', self.numberOfMatches] + args += ['--reference_labels', 'angleRot', 'angleTilt', 'ref3d', 'imageRef'] if self.useGpu: args += ['--device', 'cuda:0'] # TODO select if local > 0: From 060a36e303695acf3a0d1634737c682ab4c119b5 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 31 May 2023 12:28:09 +0200 Subject: [PATCH 080/104] Modified default local iteration count --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 3db80e1f1..a36a104cb 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -96,7 +96,7 @@ def _defineParams(self, form: Form): form.addSection(label='Global refinement') form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) - form.addParam('numberOfLocalIterations', IntParam, label='Number of local iterations', default=4) + form.addParam('numberOfLocalIterations', IntParam, label='Number of local iterations', default=1) form.addParam('numberOfAlignmentRepetitions', IntParam, label='Number of repetitions', default=2) form.addParam('maximumResolution', FloatParam, label="Maximum alignment resolution (A)", default=8.0, help='Image comparison resolution limit of the refinement') From 09f74399468b5d7ebbd714fd45979fb994f836d5 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 26 Jun 2023 09:41:15 +0200 Subject: [PATCH 081/104] Removed unused import --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index a36a104cb..287245262 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -42,7 +42,6 @@ import numpy as np from scipy import stats import itertools -import collections import os class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): From 7ae5212119eca6f5205a83767478b33eab347924 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 13 Jul 2023 08:16:15 +0000 Subject: [PATCH 082/104] Removed method parameter --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 287245262..7ebacd35c 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -442,9 +442,9 @@ def trainDatabaseStep(self, iteration: int): args += ['--max_shift', maxShift] #FIXME fails with != 190 args += ['--max_psi', maxPsi] args += ['--max_frequency', maxFrequency] - args += ['--method', 'fourier'] args += ['--training', trainingSize] - args += ['--batch', self.batchSize] + #args += ['--batch', self.batchSize] # TODO + args += ['--batch', 1024] args += ['--scratch', self._getTrainingScratchFilename()] if self.useGpu: args += ['--device', 'cuda:0'] # TODO select @@ -489,7 +489,6 @@ def alignStep(self, iteration: int, repetition: int, local: int): args += ['--rotations', nRotations] args += ['--shifts', nShift] args += ['--max_frequency', maxFrequency] - args += ['--method', 'fourier'] args += ['--dropna'] args += ['--batch', self.batchSize] args += ['--max_size', self.databaseMaximumSize] From 0b52234b654db1f2b69faba9d84f9163ccc76191 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 28 Jul 2023 11:17:25 +0000 Subject: [PATCH 083/104] Integrated the new wiener filter --- .../protocol_reconstruct_swiftres.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 7ebacd35c..e4ef17006 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -21,7 +21,8 @@ # ***************************************************************************/ from pwem.protocols import ProtRefine3D -from pwem.objects import Volume, FSC, SetOfVolumes, Class3D, SetOfParticles, SetOfClasses3D, Particle +from pwem.objects import (Volume, FSC, SetOfVolumes, Class3D, + SetOfParticles, SetOfClasses3D, Particle) from pwem.convert import transformations from pwem import emlib @@ -260,20 +261,26 @@ def is_mrc(path: str) -> bool: self.runJob('xmipp_image_convert', args, numberOfMpi=1) def correctCtfStep(self): - particles = self.inputParticles.get() + particles: SetOfParticles = self.inputParticles.get() + acquisition = particles.getAcquisition() # Perform a CTF correction using Wiener Filtering args = [] args += ['-i', self._getInputParticleMdFilename()] - args += ['-o', self._getWienerParticleStackFilename()] - args += ['--save_metadata_stack', self._getWienerParticleMdFilename()] - args += ['--sampling_rate', self._getSamplingRate()] - args += ['--pad', '2'] - args += ['--wc', -1] + args += ['-o', self._getWienerParticleMdFilename()] + args += ['--pixel_size', self._getSamplingRate()] + args += ['--spherical_aberration', acquisition.getSphericalAberration()] + args += ['--voltage', acquisition.getVoltage()] if particles.isPhaseFlipped(): args += ['--phase_flipped'] - self.runJob('xmipp_ctf_correct_wiener2d', args) + args += ['--batch', self.batchSize] + if self.useGpu: + args += ['--device', 'cuda:0'] # TODO select + + env = self.getCondaEnv() + env['LD_LIBRARY_PATH'] = '' # Torch does not like it + self.runJob('xmipp_swiftalign_wiener_2d', args, numberOfMpi=1, env=env) # Append the wiener corrected images to the second image label of the input inputMd = emlib.MetaData(self._getInputParticleMdFilename()) From be1d65937bd681d2cfe65ee172791befbc372d69 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 28 Jul 2023 11:26:07 +0000 Subject: [PATCH 084/104] Added GPU parameter --- .../protocol_reconstruct_swiftres.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index e4ef17006..e85ba46c8 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -276,7 +276,7 @@ def correctCtfStep(self): args += ['--batch', self.batchSize] if self.useGpu: - args += ['--device', 'cuda:0'] # TODO select + args += ['--device'] + self._getDeviceList() env = self.getCondaEnv() env['LD_LIBRARY_PATH'] = '' # Torch does not like it @@ -385,11 +385,6 @@ def projectVolumeStep(self, iteration: int, cls: int, repetition: int): args += ['--perturb', perturb] args += ['--sym', self.symmetryGroup] - if False: # TODO speak with coss - args += ['--compute_neighbors'] - args += ['--angular_distance', -1] - args += ['--experimental_images', self._getIterationInputParticleMdFilename(iteration)] - self.runJob('xmipp_angular_project_library', args) args = [] @@ -446,15 +441,14 @@ def trainDatabaseStep(self, iteration: int): args += ['-o', self._getTrainingIndexFilename(iteration)] args += ['--recipe', recipe] #args += ['--weights', self._getWeightsFilename(iteration)] - args += ['--max_shift', maxShift] #FIXME fails with != 190 + args += ['--max_shift', maxShift] #FIXME fails with != 180 args += ['--max_psi', maxPsi] args += ['--max_frequency', maxFrequency] args += ['--training', trainingSize] - #args += ['--batch', self.batchSize] # TODO - args += ['--batch', 1024] + args += ['--batch', self.batchSize] args += ['--scratch', self._getTrainingScratchFilename()] if self.useGpu: - args += ['--device', 'cuda:0'] # TODO select + args += ['--device'] + self._getDeviceList() if self.useFloat16: args += ['--fp16'] if self.considerInputCtf: @@ -502,7 +496,7 @@ def alignStep(self, iteration: int, repetition: int, local: int): args += ['-k', self.numberOfMatches] args += ['--reference_labels', 'angleRot', 'angleTilt', 'ref3d', 'imageRef'] if self.useGpu: - args += ['--device', 'cuda:0'] # TODO select + args += ['--device'] + self._getDeviceList() if local > 0: args += ['--local'] if self.considerInputCtf: @@ -635,7 +629,6 @@ def reconstructStep(self, iteration: int, cls: int, half: int): args += ['-i', self._getReconstructionHalfMdFilename(iteration, cls, half)] args += ['-o', self._getHalfVolumeFilename(iteration, cls, half)] args += ['--sym', self.symmetryGroup.get()] - args += ['--weight'] # TODO determine if used # Determine the execution parameters numberOfMpi = self.numberOfMpi.get() @@ -915,6 +908,11 @@ def _getOutputFscFilename(self, cls: int): def _getTrainingScratchFilename(self): return self._getTmpPath('scratch.bin') + def _getDeviceList(self): + gpus = self.getGpuList() + return list(map('cuda:'.__add__, gpus)) + + def _quaternionAverage(self, quaternions: np.ndarray) -> np.ndarray: s = np.matmul(quaternions.T, quaternions) s /= len(quaternions) From 64253acce7d60b3b01d32ebcf62bc2f903553df5 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Fri, 28 Jul 2023 11:36:18 +0000 Subject: [PATCH 085/104] Bugfix in device formatting --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index e85ba46c8..ad236c277 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -910,8 +910,7 @@ def _getTrainingScratchFilename(self): def _getDeviceList(self): gpus = self.getGpuList() - return list(map('cuda:'.__add__, gpus)) - + return list(map('cuda:{:d}'.format, gpus)) def _quaternionAverage(self, quaternions: np.ndarray) -> np.ndarray: s = np.matmul(quaternions.T, quaternions) From 0e4b38cdd3c7d78bee72ad73e50c59fcf0448b69 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 31 Jul 2023 08:32:32 +0000 Subject: [PATCH 086/104] Added optional reconstruction --- .../protocol_reconstruct_swiftres.py | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index ad236c277..dabdfbd14 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -95,6 +95,7 @@ def _defineParams(self, form: Form): help='Consider the CTF of the particles') form.addSection(label='Global refinement') + form.addParam('reconstructLast', BooleanParam, label='Reconstruct last volume', default=True) form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) form.addParam('numberOfLocalIterations', IntParam, label='Number of local iterations', default=1) form.addParam('numberOfAlignmentRepetitions', IntParam, label='Number of repetitions', default=2) @@ -181,10 +182,11 @@ def _insertIterationSteps(self, iteration: int, prerequisites): compareAnglesStepId = self._insertFunctionStep('compareAnglesStep', iteration, prerequisites=alignIds) ids = [] - for cls in range(self._getClassCount()): - reconstructIds = self._insertReconstructSteps(iteration, cls, prerequisites=alignIds) - postProcessIds = self._insertPostProcessSteps(iteration, cls, prerequisites=reconstructIds) - ids += postProcessIds + if self.reconstructLast or iteration < (self._getIterationCount() - 1): + for cls in range(self._getClassCount()): + reconstructIds = self._insertReconstructSteps(iteration, cls, prerequisites=alignIds) + postProcessIds = self._insertPostProcessSteps(iteration, cls, prerequisites=reconstructIds) + ids += postProcessIds return ids + [compareAnglesStepId] @@ -726,27 +728,29 @@ def createOutputStep(self): self._getOutputParticlesMdFilename() ) - for cls in range(self._getClassCount()): - for i in range(1, 3): - createLink( - self._getHalfVolumeFilename(lastIteration, cls, i), - self._getOutputHalfVolumeFilename(cls, i) - ) - - createLink( - self._getFilteredVolumeFilename(lastIteration, cls), - self._getOutputVolumeFilename(cls) - ) - createLink( - self._getFscFilename(lastIteration, cls), - self._getOutputFscFilename(cls) - ) - - # Create output objects - volumes = self._createOutputVolumes() + # Create output particles self._createOutputClasses3D(volumes) - self._createOutputFscs() + if self.reconstructLast: + for cls in range(self._getClassCount()): + for i in range(1, 3): + createLink( + self._getHalfVolumeFilename(lastIteration, cls, i), + self._getOutputHalfVolumeFilename(cls, i) + ) + + createLink( + self._getFilteredVolumeFilename(lastIteration, cls), + self._getOutputVolumeFilename(cls) + ) + createLink( + self._getFscFilename(lastIteration, cls), + self._getOutputFscFilename(cls) + ) + + # Create output objects + volumes = self._createOutputVolumes() + self._createOutputFscs() #--------------------------- UTILS functions -------------------------------------------- def _getIterationCount(self) -> int: From dd98570f1464e2a78c4ab781d7f6ea8bd0b4f39c Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 31 Jul 2023 08:52:04 +0000 Subject: [PATCH 087/104] Bugfix --- .../protocol_reconstruct_swiftres.py | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index dabdfbd14..2262bc7a5 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -713,24 +713,8 @@ def mergeAlignmentsStep(self, iteration: int): def createOutputStep(self): lastIteration = self._getIterationCount() - 1 - - # Link last iteration - if os.path.exists(self._getInputParticleStackFilename()): - # Use original images - alignmentMd = emlib.MetaData(self._getAlignmentMdFilename(lastIteration)) - alignmentMd.copyColumn(emlib.MDL_IMAGE, emlib.MDL_IMAGE_ORIGINAL) - alignmentMd.write(self._getOutputParticlesMdFilename()) - - else: - # Link - createLink( - self._getAlignmentMdFilename(lastIteration), - self._getOutputParticlesMdFilename() - ) - - # Create output particles - self._createOutputClasses3D(volumes) + # Create output volumes if necessary if self.reconstructLast: for cls in range(self._getClassCount()): for i in range(1, 3): @@ -751,7 +735,27 @@ def createOutputStep(self): # Create output objects volumes = self._createOutputVolumes() self._createOutputFscs() + else: + volumes = self.inputVolumes + # Link last iteration + if os.path.exists(self._getInputParticleStackFilename()): + # Use original images + alignmentMd = emlib.MetaData(self._getAlignmentMdFilename(lastIteration)) + alignmentMd.copyColumn(emlib.MDL_IMAGE, emlib.MDL_IMAGE_ORIGINAL) + alignmentMd.write(self._getOutputParticlesMdFilename()) + + else: + # Link + createLink( + self._getAlignmentMdFilename(lastIteration), + self._getOutputParticlesMdFilename() + ) + + # Create output particles + self._createOutputClasses3D(volumes) + + #--------------------------- UTILS functions -------------------------------------------- def _getIterationCount(self) -> int: return int(self.numberOfIterations) From b9405142d22ccbc96c8490ac924f3c311d53961f Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 31 Jul 2023 08:57:03 +0000 Subject: [PATCH 088/104] Removed unimplemented viewer --- xmipp3/viewers/viewer_reconstruct_swiftres.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/xmipp3/viewers/viewer_reconstruct_swiftres.py b/xmipp3/viewers/viewer_reconstruct_swiftres.py index 7a6e9a4b7..ccb2cc116 100644 --- a/xmipp3/viewers/viewer_reconstruct_swiftres.py +++ b/xmipp3/viewers/viewer_reconstruct_swiftres.py @@ -66,7 +66,6 @@ def _defineParams(self, form): form.addParam('showWeights', LabelParam, label='Display radial weight profile') form.addSection(label='Classification') - form.addParam('showClassMigration', LabelParam, label='Display class migration diagram') form.addParam('showClassSizes', LabelParam, label='Display class sizes') form.addSection(label='Angular difference from previous iteration') @@ -93,7 +92,6 @@ def _getVisualizeDict(self): 'showNoiseModel': self._showNoiseModel, 'showWeights': self._showWeights, - 'showClassMigration': self._showClassMigration, 'showClassSizes': self._showClassSizes, 'showAngleDiffMetadata': self._showAngleDiffMetadata, @@ -389,19 +387,6 @@ def _showWeights(self, e): return [fig] - def _showClassMigration(self, e): - fig, ax = plt.subplots() - - lines, points = self._computeClassMigrationElements() - - ax.add_collection(lines) - sc = ax.scatter(points[:,0], points[:,1], c=points[:,2]) - ax.set_xlabel('Iteration') - ax.set_ylabel('Class') - fig.colorbar(sc, ax=ax) - - return [fig] - def _showClassSizes(self, e): fig, ax = plt.subplots() From dbb09ee8efe88792f05de87104c11d1345848cd1 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 1 Aug 2023 08:11:46 +0000 Subject: [PATCH 089/104] Bugfix --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 2262bc7a5..96bc36fdf 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -736,7 +736,8 @@ def createOutputStep(self): volumes = self._createOutputVolumes() self._createOutputFscs() else: - volumes = self.inputVolumes + # Create a dict mimicking the base-1 ids of SetOfVolumes + volumes = dict(enumerate(self.inputVolumes, start=1)) # Link last iteration if os.path.exists(self._getInputParticleStackFilename()): From b12c7b23b9b5eef618c66015c7e74e8cfb284bf6 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 1 Aug 2023 08:18:04 +0000 Subject: [PATCH 090/104] Added pointer dereferencing --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 96bc36fdf..0a288ad35 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -22,7 +22,8 @@ from pwem.protocols import ProtRefine3D from pwem.objects import (Volume, FSC, SetOfVolumes, Class3D, - SetOfParticles, SetOfClasses3D, Particle) + SetOfParticles, SetOfClasses3D, Particle, + Pointer ) from pwem.convert import transformations from pwem import emlib @@ -737,7 +738,7 @@ def createOutputStep(self): self._createOutputFscs() else: # Create a dict mimicking the base-1 ids of SetOfVolumes - volumes = dict(enumerate(self.inputVolumes, start=1)) + volumes = dict(enumerate(map(Pointer.get, self.inputVolumes), start=1)) # Link last iteration if os.path.exists(self._getInputParticleStackFilename()): From 433f7eaa8a6f8271daa879a0172f5d70ea967caf Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 2 Aug 2023 08:57:02 +0000 Subject: [PATCH 091/104] Made wiener filtering optional --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 0a288ad35..44331a435 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -161,7 +161,7 @@ def _validate(self): def _insertAllSteps(self): convertInputStepId = self._insertFunctionStep('convertInputStep', prerequisites=[]) - if self.considerInputCtf: + if self.considerInputCtf and self.reconstructLast: correctCtfStepId = self._insertFunctionStep('correctCtfStep', prerequisites=[convertInputStepId]) lastIds = [correctCtfStepId] else: From 118c216652274e0262715f1fb600f105bf388b36 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 3 Aug 2023 06:46:17 +0000 Subject: [PATCH 092/104] Integrated precomputed tables option --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 44331a435..a6b35f22c 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -131,6 +131,8 @@ def _defineParams(self, form: Form): help='When enabled, FAISS will be prompted to use half precision floating point ' 'numbers. This may improve performance and/or memory footprint at some ' 'accuracy cost. Only supported for GPUs') + form.addParam('usePrecomputed', BooleanParam, label='Precompute centroid distances', default=False, + help='When using PQ encoding precompute pairwise distances between centroids') form.addParam('databaseTrainingSetSize', IntParam, label='Database training set size', default=int(2e6), help='Number of data-augmented particles to used when training the database') @@ -454,6 +456,8 @@ def trainDatabaseStep(self, iteration: int): args += ['--device'] + self._getDeviceList() if self.useFloat16: args += ['--fp16'] + if self.usePrecomputed: + args += ['--use_precomputed'] if self.considerInputCtf: args += ['--ctf', self._getCtfGroupInfoMdFilename(iteration)] @@ -502,6 +506,8 @@ def alignStep(self, iteration: int, repetition: int, local: int): args += ['--device'] + self._getDeviceList() if local > 0: args += ['--local'] + if self.usePrecomputed: + args += ['--use_precomputed'] if self.considerInputCtf: args += ['--ctf', self._getCtfGroupInfoMdFilename(iteration)] From c5e79ff07f3844621be2f265687109825c1252ee Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Sat, 5 Aug 2023 16:59:03 +0000 Subject: [PATCH 093/104] Added projection alignment to the output --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index a6b35f22c..25cae1687 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -1101,6 +1101,10 @@ def updateClass(cls: Class3D): itemDataIterator=itertools.chain(emlib.metadata.iterRows(particlesMd), itertools.repeat(None)) ) + # Set the alignment for the output particle set + outputParticles: SetOfParticles = classes3d.getImages() + outputParticles.setAlignmentProj() + # Define the output self._defineOutputs(**{self.OUTPUT_CLASSES_NAME: classes3d}) self._defineSourceRelation(self.inputParticles, classes3d) From ef58894d16f6709d8dfd0ac60bc3a1d5fe1833ef Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 7 Aug 2023 08:04:34 +0000 Subject: [PATCH 094/104] Settign alignment type to Class3D --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 25cae1687..cba78465f 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -1092,19 +1092,16 @@ def updateClass(cls: Class3D): clsId = cls.getObjId() representative = volumes[clsId] cls.setRepresentative(representative) + cls.setAlignmentProj() particlesMd = emlib.MetaData(self._getOutputParticlesMdFilename()) - classes3d = self._createSetOfClasses3D(self.inputParticles) + classes3d: SetOfClasses3D = self._createSetOfClasses3D(self.inputParticles) classes3d.classifyItems( updateItemCallback=updateItem, updateClassCallback=updateClass, itemDataIterator=itertools.chain(emlib.metadata.iterRows(particlesMd), itertools.repeat(None)) ) - # Set the alignment for the output particle set - outputParticles: SetOfParticles = classes3d.getImages() - outputParticles.setAlignmentProj() - # Define the output self._defineOutputs(**{self.OUTPUT_CLASSES_NAME: classes3d}) self._defineSourceRelation(self.inputParticles, classes3d) From b22b63567f95820f85cde98100bd4a16329fb8b0 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 8 Aug 2023 09:10:37 +0000 Subject: [PATCH 095/104] Added particle output for single class refinements --- .../protocol_reconstruct_swiftres.py | 113 ++++++++++++++++-- 1 file changed, 106 insertions(+), 7 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index cba78465f..e43f29329 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -23,7 +23,7 @@ from pwem.protocols import ProtRefine3D from pwem.objects import (Volume, FSC, SetOfVolumes, Class3D, SetOfParticles, SetOfClasses3D, Particle, - Pointer ) + Pointer, SetOfFSCs ) from pwem.convert import transformations from pwem import emlib @@ -36,7 +36,7 @@ createLink, cleanPattern) import xmipp3 -from xmipp3.convert import writeSetOfParticles, rowToParticle +from xmipp3.convert import readSetOfParticles, writeSetOfParticles, rowToParticle from typing import Iterable, Sequence, Optional import math @@ -48,13 +48,21 @@ class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): OUTPUT_CLASSES_NAME = 'classes' + OUTPUT_PARTICLES_NAME = 'particles' OUTPUT_VOLUMES_NAME = 'volumes' + OUTPUT_VOLUME_NAME = 'volume' + OUTPUT_FSCS_NAME = 'fscs' + OUTPUT_FSC_NAME = 'fsc' _label = 'swiftres' _conda_env = 'xmipp_swiftalign' _possibleOutputs = { OUTPUT_CLASSES_NAME: SetOfClasses3D, - OUTPUT_VOLUMES_NAME: SetOfVolumes + OUTPUT_PARTICLES_NAME: SetOfParticles, + OUTPUT_VOLUMES_NAME: SetOfVolumes, + OUTPUT_VOLUME_NAME: Volume, + OUTPUT_FSCS_NAME: SetOfFSCs, + OUTPUT_FSC_NAME: FSC } def __init__(self, **kwargs): @@ -172,7 +180,10 @@ def _insertAllSteps(self): for i in range(self._getIterationCount()): lastIds = self._insertIterationSteps(i, prerequisites=lastIds) - self._insertFunctionStep('createOutputStep', prerequisites=lastIds) + if len(self.inputVolumes) == 1: + self._insertFunctionStep('createOutputOneStep', prerequisites=lastIds) + else: + self._insertFunctionStep('createOutputMultipleStep', prerequisites=lastIds) def _insertIterationSteps(self, iteration: int, prerequisites): setupIterationStepId = self._insertFunctionStep('setupIterationStep', iteration, prerequisites=prerequisites) @@ -718,7 +729,49 @@ def mergeAlignmentsStep(self, iteration: int): self._getAlignmentMdFilename(iteration) ) - def createOutputStep(self): + def createOutputOneStep(self): + lastIteration = self._getIterationCount() - 1 + + # Create output volumes if necessary + if self.reconstructLast: + cls = 0 + for i in range(1, 3): + createLink( + self._getHalfVolumeFilename(lastIteration, cls, i), + self._getOutputHalfVolumeFilename(cls, i) + ) + + createLink( + self._getFilteredVolumeFilename(lastIteration, cls), + self._getOutputVolumeFilename(cls) + ) + createLink( + self._getFscFilename(lastIteration, cls), + self._getOutputFscFilename(cls) + ) + + # Create output objects + self._createOutputVolume(cls) + self._createOutputFsc(cls) + + # Link last iteration + if os.path.exists(self._getInputParticleStackFilename()): + # Use original images + alignmentMd = emlib.MetaData(self._getAlignmentMdFilename(lastIteration)) + alignmentMd.copyColumn(emlib.MDL_IMAGE, emlib.MDL_IMAGE_ORIGINAL) + alignmentMd.write(self._getOutputParticlesMdFilename()) + + else: + # Link + createLink( + self._getAlignmentMdFilename(lastIteration), + self._getOutputParticlesMdFilename() + ) + + # Create output particles + self._createOutputSetOfParticles() + + def createOutputMultipleStep(self): lastIteration = self._getIterationCount() - 1 # Create output volumes if necessary @@ -1109,6 +1162,17 @@ def updateClass(cls: Class3D): return classes3d + def _createOutputSetOfParticles(self): + particles = self._createSetOfParticles() + readSetOfParticles(self._getOutputParticlesMdFilename(), particles) + + # Define the output + self._defineOutputs(**{self.OUTPUT_PARTICLES_NAME: particles}) + self._defineSourceRelation(self.inputParticles, particles) + self._defineSourceRelation(self.inputVolumes, particles) + + return particles + def _createOutputVolumes(self): volumes = self._createSetOfVolumes() volumes.setSamplingRate(self._getSamplingRate()) @@ -1126,12 +1190,30 @@ def _createOutputVolumes(self): volumes.append(volume) # Define the output - self._defineOutputs(outputVolumes=volumes) + self._defineOutputs(**{self.OUTPUT_VOLUMES_NAME: volumes}) self._defineSourceRelation(self.inputParticles, volumes) self._defineSourceRelation(self.inputVolumes, volumes) return volumes + def _createOutputVolume(self, cls: int = 0): + volume=Volume(objId=cls+1) + + # Fill + volume.setFileName(self._getOutputVolumeFilename(cls)) + volume.setHalfMaps([ + self._getOutputHalfVolumeFilename(cls, 1), + self._getOutputHalfVolumeFilename(cls, 2), + ]) + volume.setSamplingRate(self._getSamplingRate()) + + # Define the output + self._defineOutputs(**{self.OUTPUT_VOLUME_NAME: volume}) + self._defineSourceRelation(self.inputParticles, volume) + self._defineSourceRelation(self.inputVolumes, volume) + + return volume + def _createOutputFscs(self): fscs = self._createSetOfFSCs() @@ -1148,12 +1230,29 @@ def _createOutputFscs(self): fscs.append(fsc) # Define the output - self._defineOutputs(outputFSC=fscs) + self._defineOutputs(**{self.OUTPUT_FSCS_NAME: fscs}) self._defineSourceRelation(self.inputParticles, fscs) self._defineSourceRelation(self.inputVolumes, fscs) return fscs + def _createOutputFsc(self, cls: int = 0): + fsc = FSC(objId=cls+1) + + # Load from metadata + fsc.loadFromMd( + self._getOutputFscFilename(cls), + emlib.MDL_RESOLUTION_FREQ, + emlib.MDL_RESOLUTION_FRC + ) + + # Define the output + self._defineOutputs(**{self.OUTPUT_FSC_NAME: fsc}) + self._defineSourceRelation(self.inputParticles, fsc) + self._defineSourceRelation(self.inputVolumes, fsc) + + return fsc + def _mergeMetadata(self, src: Iterable[str], dst: str): it = iter(src) From d929bb79ea1c5997b0957cbc18912ea3d31ff6e8 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 9 Aug 2023 09:01:10 +0000 Subject: [PATCH 096/104] Added missing information to the output set of particles --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index e43f29329..7b6cf53bd 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -1163,7 +1163,8 @@ def updateClass(cls: Class3D): return classes3d def _createOutputSetOfParticles(self): - particles = self._createSetOfParticles() + particles: SetOfParticles = self._createSetOfParticles() + particles.copyInfo(self.inputParticles) readSetOfParticles(self._getOutputParticlesMdFilename(), particles) # Define the output From 7235a8bee7d15d926761683a8b64805f9b095afe Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 9 Aug 2023 09:24:57 +0000 Subject: [PATCH 097/104] Fixed error --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 7b6cf53bd..4bab4c2ee 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -1164,7 +1164,7 @@ def updateClass(cls: Class3D): def _createOutputSetOfParticles(self): particles: SetOfParticles = self._createSetOfParticles() - particles.copyInfo(self.inputParticles) + particles.copyInfo(self.inputParticles.get()) readSetOfParticles(self._getOutputParticlesMdFilename(), particles) # Define the output From 92ae5150e0ee11121f32ae67d29cc30cad956eab Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Mon, 28 Aug 2023 09:19:31 +0000 Subject: [PATCH 098/104] Moved extra labels --- .../protocol_reconstruct_swiftres.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 4bab4c2ee..5f4f1a123 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -54,6 +54,15 @@ class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): OUTPUT_FSCS_NAME = 'fscs' OUTPUT_FSC_NAME = 'fsc' + OUTPUT_EXTRA_LABELS = [ + #emlib.MDL_COST, + #emlib.MDL_WEIGHT, + #emlib.MDL_CORRELATION_IDX, + #emlib.MDL_CORRELATION_MASK, + #emlib.MDL_CORRELATION_WEIGHT, + #emlib.MDL_IMED + ] + _label = 'swiftres' _conda_env = 'xmipp_swiftalign' _possibleOutputs = { @@ -1125,18 +1134,9 @@ def _getIterationMaxShift(self, iteration: int) -> float: return float(self.initialMaxShift) def _createOutputClasses3D(self, volumes: SetOfVolumes): - EXTRA_LABELS = [ - #emlib.MDL_COST, - #emlib.MDL_WEIGHT, - #emlib.MDL_CORRELATION_IDX, - #emlib.MDL_CORRELATION_MASK, - #emlib.MDL_CORRELATION_WEIGHT, - #emlib.MDL_IMED - ] - def updateItem(item: Particle, row: emlib.metadata.Row): if row is not None: - particle: Particle = rowToParticle(row, extraLabels=EXTRA_LABELS) + particle: Particle = rowToParticle(row, extraLabels=self.EXTRA_LABELS) item.copy(particle) else: item._appendItem = False @@ -1165,7 +1165,7 @@ def updateClass(cls: Class3D): def _createOutputSetOfParticles(self): particles: SetOfParticles = self._createSetOfParticles() particles.copyInfo(self.inputParticles.get()) - readSetOfParticles(self._getOutputParticlesMdFilename(), particles) + readSetOfParticles(self._getOutputParticlesMdFilename(), particles, extraLabels=self.EXTRA_LABELS) # Define the output self._defineOutputs(**{self.OUTPUT_PARTICLES_NAME: particles}) From 63bbb20141eb3c50c7162d95e321b58b4e8978b1 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Tue, 29 Aug 2023 12:04:43 +0000 Subject: [PATCH 099/104] Bugfix --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 5f4f1a123..456e63418 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -1136,7 +1136,7 @@ def _getIterationMaxShift(self, iteration: int) -> float: def _createOutputClasses3D(self, volumes: SetOfVolumes): def updateItem(item: Particle, row: emlib.metadata.Row): if row is not None: - particle: Particle = rowToParticle(row, extraLabels=self.EXTRA_LABELS) + particle: Particle = rowToParticle(row, extraLabels=self.OUTPUT_EXTRA_LABELS) item.copy(particle) else: item._appendItem = False @@ -1165,7 +1165,7 @@ def updateClass(cls: Class3D): def _createOutputSetOfParticles(self): particles: SetOfParticles = self._createSetOfParticles() particles.copyInfo(self.inputParticles.get()) - readSetOfParticles(self._getOutputParticlesMdFilename(), particles, extraLabels=self.EXTRA_LABELS) + readSetOfParticles(self._getOutputParticlesMdFilename(), particles, extraLabels=self.OUTPUT_EXTRA_LABELS) # Define the output self._defineOutputs(**{self.OUTPUT_PARTICLES_NAME: particles}) From 8e41c41eadeeab7b3737c3bd778e32700a2fa2f9 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 30 Aug 2023 09:56:00 +0000 Subject: [PATCH 100/104] Added gradient descent parameters --- .../protocol_reconstruct_swiftres.py | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 456e63418..0df35f68d 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -30,7 +30,8 @@ from pyworkflow.protocol.params import (Form, PointerParam, FloatParam, IntParam, StringParam, BooleanParam, - MultiPointerParam, + MultiPointerParam, EnumParam, + GT, GE, Range, LEVEL_ADVANCED, USE_GPU, GPU_LIST ) from pyworkflow.utils.path import (cleanPath, makePath, copyFile, moveFile, createLink, cleanPattern) @@ -63,6 +64,11 @@ class XmippProtReconstructSwiftres(ProtRefine3D, xmipp3.XmippProtocol): #emlib.MDL_IMED ] + GRADIENT_DESCENT_OPTIONS = [ + 'None', + 'RMSProp' + ] + _label = 'swiftres' _conda_env = 'xmipp_swiftalign' _possibleOutputs = { @@ -102,7 +108,7 @@ def _defineParams(self, form: Form): form.addParam('symmetryGroup', StringParam, default='c1', label='Symmetry group', help='If no symmetry is present, give c1') - form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0, + form.addParam('initialResolution', FloatParam, label="Initial resolution (A)", default=10.0, validators=[GT(0)], help='Image comparison resolution limit at the first iteration of the refinement') form.addParam('mask', PointerParam, label="Mask", pointerClass='VolumeMask', allowsNull=True, help='The mask values must be between 0 (remove these pixels) and 1 (let them pass). Smooth masks are recommended.') @@ -112,31 +118,43 @@ def _defineParams(self, form: Form): default=True, help='Consider the CTF of the particles') - form.addSection(label='Global refinement') + form.addSection(label='Refinement') form.addParam('reconstructLast', BooleanParam, label='Reconstruct last volume', default=True) - form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3) - form.addParam('numberOfLocalIterations', IntParam, label='Number of local iterations', default=1) - form.addParam('numberOfAlignmentRepetitions', IntParam, label='Number of repetitions', default=2) - form.addParam('maximumResolution', FloatParam, label="Maximum alignment resolution (A)", default=8.0, + form.addParam('numberOfIterations', IntParam, label='Number of iterations', default=3, validators=[GT(0)]) + form.addParam('numberOfLocalIterations', IntParam, label='Number of local iterations', default=1, validators=[GT(0)]) + form.addParam('numberOfAlignmentRepetitions', IntParam, label='Number of repetitions', default=2, validators=[GT(0)]) + form.addParam('maximumResolution', FloatParam, label="Maximum alignment resolution (A)", default=8.0, validators=[GT(0)], help='Image comparison resolution limit of the refinement') - form.addParam('nextResolutionCriterion', FloatParam, label="FSC criterion", default=0.5, + form.addParam('nextResolutionCriterion', FloatParam, label="FSC criterion", default=0.5, validators=[Range(0, 1)], expertLevel=LEVEL_ADVANCED, help='The resolution of the reconstruction is defined as the inverse of the frequency at which '\ 'the FSC drops below this value. Typical values are 0.143 and 0.5' ) - form.addParam('initialMaxPsi', FloatParam, label='Maximum psi (deg)', default=180.0, + form.addParam('initialMaxPsi', FloatParam, label='Maximum psi (deg)', default=180.0, validators=[Range(0, 180)], expertLevel=LEVEL_ADVANCED, help='Maximum psi parameter of the particles') - form.addParam('initialMaxShift', FloatParam, label='Maximum shift (px)', default=16.0, + form.addParam('initialMaxShift', FloatParam, label='Maximum shift (px)', default=16.0, validators=[GT(0)], help='Maximum shift of the particle in pixels') form.addParam('useAutomaticStep', BooleanParam, label='Use automatic step', default=True, expertLevel=LEVEL_ADVANCED, help='Automatically determine the step used when exploring the projection landscape') stepGroup = form.addGroup('Steps', condition='not useAutomaticStep', expertLevel=LEVEL_ADVANCED) - stepGroup.addParam('angleStep', FloatParam, label='Angle step (deg)', default=5.0) - stepGroup.addParam('shiftStep', FloatParam, label='Shift step (px)', default=2.0) - form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, + stepGroup.addParam('angleStep', FloatParam, label='Angle step (deg)', default=5.0, validators=[GT(0)]) + stepGroup.addParam('shiftStep', FloatParam, label='Shift step (px)', default=2.0, validators=[GT(0)]) + + form.addParam('gradientDescent', EnumParam, label='Gradient descent', choices=self.GRADIENT_DESCENT_OPTIONS, + default=0, + help='If selected, each iteration will be subdivided in multiple iterations of a gradient descent') + form.addParam('minibatchSize', IntParam, label='Mini-batch size', default=16384, + condition='gradientDescent > 0', validators=[GT(0)], + help='Size of the minibatch when performing a gradient descent') + rmspropGroup = form.addGroup('RMSProp', condition='gradientDescent == 1', expertLevel=LEVEL_ADVANCED) + rmspropGroup.addParam('rmspropGamma', FloatParam, label='Gradient learning rate', default=0.9, validators=[Range(0,1)]) + rmspropGroup.addParam('rmspropNu', FloatParam, label='Learning rate', default=0.001, validators=[GE(0)]) + rmspropGroup.addParam('rmspropEps', FloatParam, label='Stability factor', default=1e-8, validators=[GE(0)]) + + form.addParam('reconstructPercentage', FloatParam, label='Reconstruct percentage (%)', default=50, validators=[Range(0,100)], help='Percentage of best particles used for reconstruction') - form.addParam('numberOfMatches', IntParam, label='Number of matches', default=1, + form.addParam('numberOfMatches', IntParam, label='Number of matches', default=1, validators=[GE(0)], help='Number of reference matches for each particle') form.addSection(label='Compute') From 71bdd685a16481a4c533d28ede13c923214d920d Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Wed, 30 Aug 2023 10:10:50 +0000 Subject: [PATCH 101/104] Added rmsprop step --- .../protocol_reconstruct_swiftres.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 0df35f68d..f716b065b 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -717,6 +717,19 @@ def reconstructStep(self, iteration: int, cls: int, half: int): # Run self.runJob(reconstructProgram, args, numberOfMpi=numberOfMpi) + def rmspropGradientDescentStep(self, iteration: int, cls: int, half: int): + args = [] + args += ['--map', self._getHalfVolumeFilename(iteration, cls, half)] + args += ['--rec', self._getReconstructionHalfVolumeFilename(iteration, cls, half)] + args += ['--sigma2', self._getHalfGradientFilename(iteration, cls, half)] + args += ['--gamma', self.rmspropGamma] + args += ['--nu', self.rmspropNu] + args += ['--eps', self.rmspropEps] + args += ['--omap', self._getHalfVolumeFilename(iteration, cls, half)] + args += ['--osigma2', self._getHalfGradientFilename(iteration, cls, half)] + + self.runJob('xmipp_rms_prop_reconstruction', args, numberOfMpi=1) + def computeFscStep(self, iteration: int, cls: int): args = [] args += ['--ref', self._getHalfVolumeFilename(iteration, cls, 1)] @@ -975,7 +988,13 @@ def _getFilteredReconstructionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'well_aligned.xmd') def _getReconstructionHalfMdFilename(self, iteration: int, cls: int, half: int): - return self._getClassPath(iteration, cls, 'aligned%06d.xmd' % half) + return self._getClassPath(iteration, cls, 'aligned_half%01d.xmd' % half) + + def _getReconstructionHalfVolumeFilename(self, iteration: int, cls: int, half: int): + return self._getClassPath(iteration, cls, 'reconstruction_half%01d.mrc' % half) + + def _getHalfGradientFilename(self, iteration: int, cls: int, half: int): + return self._getClassPath(iteration, cls, 'gradient_half%01d.mrc' % half) def _getHalfVolumeFilename(self, iteration: int, cls: int, half: int): return self._getClassPath(iteration, cls, 'volume_half%01d.mrc' % half) From 1cc62a0f6d57484e2fdf2359107e880e046a1e9f Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 31 Aug 2023 09:16:33 +0000 Subject: [PATCH 102/104] Fixed rmsprop call --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index f716b065b..7063be33d 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -728,7 +728,7 @@ def rmspropGradientDescentStep(self, iteration: int, cls: int, half: int): args += ['--omap', self._getHalfVolumeFilename(iteration, cls, half)] args += ['--osigma2', self._getHalfGradientFilename(iteration, cls, half)] - self.runJob('xmipp_rms_prop_reconstruction', args, numberOfMpi=1) + self.runJob('xmipp_rmsprop_reconstruction', args, numberOfMpi=1) def computeFscStep(self, iteration: int, cls: int): args = [] From 0aad568cdf033affc61c3722952746f1e6546543 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 16 May 2024 10:28:36 +0000 Subject: [PATCH 103/104] Renaming reconstruct input file --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 7063be33d..298ac3109 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -670,6 +670,12 @@ def splitStep(self, iteration: int, cls: int): args += ['-n', 2] self.runJob('xmipp_metadata_split', args, numberOfMpi=1) + + for half in (1, 2): + moveFile( + self._getClassPath(iteration, cls, 'aligned%6d.xmd' % half), + self._getReconstructionHalfMdFilename(iteration, cls, half) + ) def reconstructStep(self, iteration: int, cls: int, half: int): args = [] From 1909f3faaaa193bc55071399bf4d8984370d3631 Mon Sep 17 00:00:00 2001 From: Oier Lauzirika Zarrabeitia Date: Thu, 16 May 2024 10:32:44 +0000 Subject: [PATCH 104/104] Fix --- xmipp3/protocols/protocol_reconstruct_swiftres.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/xmipp3/protocols/protocol_reconstruct_swiftres.py b/xmipp3/protocols/protocol_reconstruct_swiftres.py index 298ac3109..37601b77e 100644 --- a/xmipp3/protocols/protocol_reconstruct_swiftres.py +++ b/xmipp3/protocols/protocol_reconstruct_swiftres.py @@ -670,12 +670,6 @@ def splitStep(self, iteration: int, cls: int): args += ['-n', 2] self.runJob('xmipp_metadata_split', args, numberOfMpi=1) - - for half in (1, 2): - moveFile( - self._getClassPath(iteration, cls, 'aligned%6d.xmd' % half), - self._getReconstructionHalfMdFilename(iteration, cls, half) - ) def reconstructStep(self, iteration: int, cls: int, half: int): args = [] @@ -994,7 +988,7 @@ def _getFilteredReconstructionMdFilename(self, iteration: int, cls: int): return self._getClassPath(iteration, cls, 'well_aligned.xmd') def _getReconstructionHalfMdFilename(self, iteration: int, cls: int, half: int): - return self._getClassPath(iteration, cls, 'aligned_half%01d.xmd' % half) + return self._getClassPath(iteration, cls, 'aligned%06d.xmd' % half) def _getReconstructionHalfVolumeFilename(self, iteration: int, cls: int, half: int): return self._getClassPath(iteration, cls, 'reconstruction_half%01d.mrc' % half)