Resolves #862; one.util.ensure_list -> iblutil.util.ensure_list

ibllib/io/extractors/mesoscope.py

@@ -4,8 +4,8 @@
 import numpy as np
 from scipy.signal import find_peaks
 import one.alf.io as alfio
-from one.util import ensure_list
 from one.alf.files import session_path_parts
+from iblutil.util import ensure_list
 import matplotlib.pyplot as plt
 from packaging import version
 

ibllib/oneibl/data_handlers.py

@@ -15,10 +15,10 @@
 
 from one.api import ONE
 from one.webclient import AlyxClient
-from one.util import filter_datasets, ensure_list
+from one.util import filter_datasets
 from one.alf.files import add_uuid_string, session_path_parts
 from one.alf.cache import _make_datasets_df
-from iblutil.util import flatten
+from iblutil.util import flatten, ensure_list
 
 from ibllib.oneibl.registration import register_dataset, get_lab, get_local_data_repository
 from ibllib.oneibl.patcher import FTPPatcher, SDSCPatcher, SDSC_ROOT_PATH, SDSC_PATCH_PATH

ibllib/oneibl/patcher.py

@@ -33,14 +33,14 @@
 
 import globus_sdk
 import iblutil.io.params as iopar
+from iblutil.util import ensure_list
 from one.alf.files import get_session_path, add_uuid_string
 from one.alf.spec import is_uuid_string, is_uuid
 from one import params
 from one.webclient import AlyxClient
 from one.converters import path_from_dataset
 from one.remote import globus
 from one.remote.aws import url2uri
-from one.util import ensure_list
 
 from ibllib.oneibl.registration import register_dataset
 

ibllib/oneibl/registration.py

@@ -12,8 +12,8 @@
 from one.webclient import AlyxClient, no_cache
 from one.converters import ConversionMixin
 import one.alf.exceptions as alferr
-from one.util import datasets2records, ensure_list
 from one.api import ONE
+from iblutil.util import ensure_list
 
 import ibllib
 import ibllib.io.extractors.base

ibllib/pipes/base_tasks.py

@@ -4,8 +4,7 @@
 
 from packaging import version
 from one.webclient import no_cache
-from one.util import ensure_list
-from iblutil.util import flatten
+from iblutil.util import flatten, ensure_list
 import matplotlib.image
 from skimage.io import ImageCollection, imread
 
@@ -551,7 +550,7 @@ def register_snapshots(self, unlink=False, collection=None):
                 snapshot = self._save_as_png(snapshot_tif := snapshot)
                 if unlink:
                     snapshot_tif.unlink()
-            _logger.debug('Uploading "%s"...', snapshot.relative_to(self.session_path))
+            _logger.info('Uploading "%s"...', snapshot.relative_to(self.session_path))
             if snapshot.with_suffix('.txt').exists():
                 with open(snapshot.with_suffix('.txt'), 'r') as txt_file:
                     note['text'] = txt_file.read().strip()

ibllib/pipes/mesoscope_tasks.py

@@ -34,7 +34,7 @@
 from one.alf.spec import to_alf
 from one.alf.files import filename_parts, session_path_parts
 import one.alf.exceptions as alferr
-from iblutil.util import flatten
+from iblutil.util import flatten, ensure_list
 from iblatlas.atlas import ALLEN_CCF_LANDMARKS_MLAPDV_UM, MRITorontoAtlas
 
 from ibllib.pipes import base_tasks
@@ -489,7 +489,7 @@ def _consolidate_exptQC(exptQC):
 
         # Merge and make sure same indexes have same names across all files
         frameQC_names_list = [e['frameQC_names'] for e in exptQC]
-        frameQC_names_list = [{f: 0} if isinstance(f, str) else {f[i]: i for i in range(len(f))}
+        frameQC_names_list = [{k: i for i, k in enumerate(ensure_list(f)) if any(k)}
                               for f in frameQC_names_list]
         frameQC_names = {k: v for d in frameQC_names_list for k, v in d.items()}
         for d in frameQC_names_list:

ibllib/pipes/tasks.py

@@ -85,10 +85,9 @@
 from ibllib.oneibl import data_handlers
 from ibllib.oneibl.data_handlers import get_local_data_repository
 from ibllib.oneibl.registration import get_lab
-from iblutil.util import Bunch, flatten
+from iblutil.util import Bunch, flatten, ensure_list
 import one.params
 from one.api import ONE
-from one.util import ensure_list
 from one import webclient
 import one.alf.io as alfio
 

ibllib/tests/test_mesoscope.py

@@ -97,6 +97,38 @@ def test_get_default_tau(self):
             subject_detail['genotype'].pop(1)
             self.assertEqual(self.task.get_default_tau(), 1.5)  # return the default value
 
+    def test_consolidate_exptQC(self):
+        """Test for MesoscopePreprocess._consolidate_exptQC method."""
+        exptQC = [
+            {'frameQC_names': np.array(['ok', 'PMT off', 'galvos fault', 'high signal'], dtype=object),
+             'frameQC_frames': np.array([0, 0, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4])},
+            {'frameQC_names': np.array(['ok', 'PMT off', 'galvos fault', 'high signal'], dtype=object),
+             'frameQC_frames': np.zeros(50, dtype=int)}
+        ]
+
+        # Check concatinates frame QC arrays
+        frameQC, frameQC_names, bad_frames = self.task._consolidate_exptQC(exptQC)
+        expected_frames = np.r_[exptQC[0]['frameQC_frames'], exptQC[1]['frameQC_frames']]
+        np.testing.assert_array_equal(expected_frames, frameQC)
+        expected = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+        np.testing.assert_array_equal(expected, bad_frames)
+        self.assertCountEqual(['qc_values', 'qc_labels'], frameQC_names.columns)
+        self.assertCountEqual(range(4), frameQC_names['qc_values'])
+        expected = ['ok', 'PMT off', 'galvos fault', 'high signal']
+        self.assertCountEqual(expected, frameQC_names['qc_labels'])
+
+        # Check with empty array (happens sometimes when converting .mat to .npy)
+        exptQC[1]['frameQC_names'] = np.r_[exptQC[1]['frameQC_names'], np.array([], dtype='<U1')]
+        # Check with single str instead of array
+        exptQC[1]['frameQC_names'] = 'ok'
+        frameQC, frameQC_names, bad_frames = self.task._consolidate_exptQC(exptQC)
+        self.assertCountEqual(expected, frameQC_names['qc_labels'])
+        np.testing.assert_array_equal(expected_frames, frameQC)
+        # Check with inconsistent enumerations
+        exptQC[0]['frameQC_names'] = expected
+        exptQC[1]['frameQC_names'] = [*expected[-2:], *expected[:-2]]
+        self.assertRaises(IOError, self.task._consolidate_exptQC, exptQC)
+
     def test_setup_uncompressed(self):
         """Test set up behaviour when raw tifs present."""
         # Test signature when clobber = True

requirements.txt

@@ -24,7 +24,7 @@ tqdm>=4.32.1
 # ibl libraries
 iblatlas>=0.5.3
 ibl-neuropixel>=1.0.1
-iblutil>=1.11.0
+iblutil>=1.13.0
 mtscomp>=1.0.1
 ONE-api~=2.9.rc0
 phylib>=2.6.0