Fall back to void dtype if extended header interpretation fails

#11

CHANGELOG.txt

@@ -9,6 +9,7 @@ For next release
 * Speed up mrcfile-header command line tool by reading headers only, not data
 * Allow opening of files with incorrect machine stamps in permissive mode
 * Fix bug in validation of exttyp field in Python 3 (thanks to Holger Kohr)
+* Fix bug in opening files with misleading exttyp information
 * Update documentation to point to Python 3.6 instead of 2.7, and fix all
   cross-references
 * Other documentation improvements

mrcfile/load_functions.py

@@ -113,6 +113,9 @@ def open(name, mode='r', permissive=False, header_only=False):  # @ReservedAssig
             block is longer than expected from the dimensions in the header.
         RuntimeWarning: If the file is not a valid MRC file and ``permissive``
             is :data:`True`.
+        RuntimeWarning: If the header's ``exttyp`` field is set to a known
+            value but the extended header's size is not a multiple of the
+            number of bytes in the corresponding dtype.
     """
     NewMrc = MrcFile
     if os.path.exists(name):

mrcfile/mrcfile.py

@@ -75,15 +75,23 @@ def __init__(self, name, mode='r', overwrite=False, permissive=False,
         
         Raises:
             :exc:`ValueError`: If the mode is not one of ``r``, ``r+`` or
-                ``w+``, the file is not a valid MRC file, or if the mode is
-                ``w+``, the file already exists and overwrite is :data:`False`.
+                ``w+``.
+            :exc:`ValueError`: If the file is not a valid MRC file and
+                ``permissive`` is :data:`False`.
+            :exc:`ValueError`: If the mode is ``w+``, the file already exists
+                and overwrite is :data:`False`.
             :exc:`OSError`: If the mode is ``r`` or ``r+`` and the file does
                 not exist.
         
         Warns:
-            RuntimeWarning: The file appears to be a valid MRC file but the
+            RuntimeWarning: If the file appears to be a valid MRC file but the
                 data block is longer than expected from the dimensions in the
                 header.
+            RuntimeWarning: If the file is not a valid MRC file and
+                ``permissive`` is :data:`True`.
+            RuntimeWarning: If the header's ``exttyp`` field is set to a known
+                value but the extended header's size is not a multiple of the
+                number of bytes in the corresponding dtype.
         """
         super(MrcFile, self).__init__(permissive=permissive, **kwargs)
 

mrcfile/mrcinterpreter.py

@@ -98,8 +98,17 @@ def __init__(self, iostream=None, permissive=False, header_only=False,
                 file. The default is :data:`False`.
         
         Raises:
-            :exc:`ValueError`: If ``iostream`` is given and the data it
-                contains cannot be interpreted as a valid MRC file.
+            :exc:`ValueError`: If ``iostream`` is given, the data it contains
+                cannot be interpreted as a valid MRC file and ``permissive``
+                is :data:`False`.
+
+        Warns:
+            RuntimeWarning: If ``iostream`` is given, the data it contains
+                cannot be interpreted as a valid MRC file and ``permissive``
+                is :data:`True`.
+            RuntimeWarning: If the header's ``exttyp`` field is set to a known
+                value but the extended header's size is not a multiple of the
+                number of bytes in the corresponding dtype.
         """
         super(MrcInterpreter, self).__init__(**kwargs)
 
@@ -151,7 +160,8 @@ def _read(self, header_only=False):
                 stream. The default is :data:`False`.
         
         Raises:
-            :exc:`ValueError`: If the file is not a valid MRC file.
+            :exc:`ValueError`: If the data in the stream cannot be interpreted
+                 as a valid MRC file.
         """
         self._read_header()
         self._read_extended_header()
@@ -165,7 +175,12 @@ def _read_header(self):
         stream will be advanced by 1024 bytes.
         
         Raises:
-            :exc:`ValueError`: If the file is not a valid MRC file.
+            :exc:`ValueError`: If the data in the stream cannot be interpreted
+                 as a valid MRC file. and ``permissive`` is :data:`False`.
+
+        Warns:
+            RuntimeWarning:  If the data in the stream cannot be interpreted
+                 as a valid MRC file. and ``permissive`` is :data:`True`.
         """
         # Read 1024 bytes from the stream
         header_str = self._iostream.read(HEADER_DTYPE.itemsize)
@@ -234,15 +249,24 @@ def _read_extended_header(self):
         If the extended header is recognised as FEI microscope metadata (by
         'FEI1' in the header's ``exttyp`` field), its dtype is set
         appropriately. Otherwise, the dtype is set as void (``'V1'``).
+
+        Warns:
+            RuntimeWarning: If the header's ``exttyp`` field is set to 'FEI1'
+                but the extended header's size is not a multiple of the number
+                of bytes in the FEI metadata dtype.
         """
         ext_header_str = self._iostream.read(int(self.header.nsymbt))
-
-        if self.header['exttyp'] == b'FEI1':
-            dtype = FEI_EXTENDED_HEADER_DTYPE
-        else:
-            dtype = 'V1'
-
-        self._extended_header = np.frombuffer(ext_header_str, dtype=dtype)
+
+        self._extended_header = np.frombuffer(ext_header_str, dtype='V1')
+
+        try:
+            if self.header.exttyp == b'FEI1':
+                self._extended_header.dtype = FEI_EXTENDED_HEADER_DTYPE
+        except ValueError:
+            warnings.warn("File has exttyp '{}' but the extended header "
+                          "cannot be interpreted as that type"
+                          .format(self.header.exttyp), RuntimeWarning)
+
         self._extended_header.flags.writeable = not self._read_only
 
     def _read_data(self):

tests/test_mrcfile.py

@@ -321,6 +321,24 @@ def test_removing_extended_header(self):
             assert mrc.header.nsymbt == 0
             file_size = mrc._iostream.tell() # relies on flush() leaving stream at end
             assert file_size == mrc.header.nbytes + mrc.data.nbytes
+
+    def test_extended_header_with_incorrect_type(self):
+        data = np.arange(12, dtype=np.int16).reshape(3, 4)
+        extended_header = np.array('example extended header', dtype='S')
+        with self.newmrc(self.temp_mrc_name, mode='w+') as mrc:
+            mrc.set_data(data)
+            mrc.set_extended_header(extended_header)
+            mrc.header.exttyp = b'FEI1'
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            with self.newmrc(self.temp_mrc_name, mode='r+') as mrc:
+                # Test that the file is still read, and the dtype falls back to 'V'
+                assert mrc.extended_header.dtype.kind == 'V'
+                mrc.extended_header.dtype = 'S{}'.format(mrc.extended_header.nbytes)
+                np.testing.assert_array_equal(mrc.extended_header, extended_header)
+            assert len(w) == 1
+            assert "FEI1" in str(w[0].message)
+            assert "extended header" in str(w[0].message)
 
     def test_can_edit_data_in_read_write_mode(self):
         with self.newmrc(self.temp_mrc_name, mode='w+') as mrc: