Skip to content

Commit

Permalink
Merge pull request #814 from mantidproject/807_parse_imat_csv
Browse files Browse the repository at this point in the history
  • Loading branch information
Dimitar Tasev authored Jan 15, 2021
2 parents 0603620 + 7270a24 commit c5c5e9e
Show file tree
Hide file tree
Showing 9 changed files with 195 additions and 121 deletions.
75 changes: 31 additions & 44 deletions mantidimaging/core/data/test/fake_logfile.py
View file
Original file line number Diff line number Diff line change
@@ -1,53 +1,40 @@
# Copyright (C) 2020 ISIS Rutherford Appleton Laboratory UKRI
# SPDX - License - Identifier: GPL-3.0-or-later

from mantidimaging.core.utility.imat_log_file_parser import IMATLogFile, EXPECTED_HEADER_FOR_IMAT_LOG_FILE
from mantidimaging.core.utility.imat_log_file_parser import CSVLogParser, IMATLogFile, TextLogParser


def generate_logfile() -> IMATLogFile:
def generate_txt_logfile() -> IMATLogFile:
data = [
EXPECTED_HEADER_FOR_IMAT_LOG_FILE, # checked if exists, but skipped
[""], # skipped when parsing
TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, # checked if exists, but skipped
"", # skipped when parsing
# for each row a list with 4 entries is currently expected
[
"Sun Feb 10 00:22:04 2019", "Projection: 0 angle: 0.0", "Monitor 3 before: 4577907",
"Monitor 3 after: 4720271"
],
[
"Sun Feb 10 00:22:37 2019", "Projection: 1 angle: 0.3152", "Monitor 3 before: 4729337",
"Monitor 3 after: 4871319"
],
[
"Sun Feb 10 00:23:10 2019", "Projection: 2 angle: 0.6304", "Monitor 3 before: 4879923",
"Monitor 3 after: 5022689"
],
[
"Sun Feb 10 00:23:43 2019", "Projection: 3 angle: 0.9456", "Monitor 3 before: 5031423",
"Monitor 3 after: 5172216"
],
[
"Sun Feb 10 00:24:16 2019", "Projection: 4 angle: 1.2608", "Monitor 3 before: 5180904",
"Monitor 3 after: 5322691"
],
[
"Sun Feb 10 00:24:49 2019", "Projection: 5 angle: 1.576", "Monitor 3 before: 5334225",
"Monitor 3 after: 5475239"
],
[
"Sun Feb 10 00:25:22 2019", "Projection: 6 angle: 1.8912", "Monitor 3 before: 5483964",
"Monitor 3 after: 5626608"
],
[
"Sun Feb 10 00:25:55 2019", "Projection: 7 angle: 2.2064", "Monitor 3 before: 5635673",
"Monitor 3 after: 5777316"
],
[
"Sun Feb 10 00:26:29 2019", "Projection: 8 angle: 2.5216", "Monitor 3 before: 5786535",
"Monitor 3 after: 5929002"
],
[
"Sun Feb 10 00:27:02 2019", "Projection: 9 angle: 2.8368", "Monitor 3 before: 5938142",
"Monitor 3 after: 6078866"
]
"Sun Feb 10 00:22:04 2019 Projection: 0 angle: 0.0 Monitor 3 before: 4577907 Monitor 3 after: 4720271", # noqa: E501
"Sun Feb 10 00:22:37 2019 Projection: 1 angle: 0.3152 Monitor 3 before: 4729337 Monitor 3 after: 4871319", # noqa: E501
"Sun Feb 10 00:23:10 2019 Projection: 2 angle: 0.6304 Monitor 3 before: 4879923 Monitor 3 after: 5022689", # noqa: E501
"Sun Feb 10 00:23:43 2019 Projection: 3 angle: 0.9456 Monitor 3 before: 5031423 Monitor 3 after: 5172216", # noqa: E501
"Sun Feb 10 00:24:16 2019 Projection: 4 angle: 1.2608 Monitor 3 before: 5180904 Monitor 3 after: 5322691", # noqa: E501
"Sun Feb 10 00:24:49 2019 Projection: 5 angle: 1.576 Monitor 3 before: 5334225 Monitor 3 after: 5475239", # noqa: E501
"Sun Feb 10 00:25:22 2019 Projection: 6 angle: 1.8912 Monitor 3 before: 5483964 Monitor 3 after: 5626608", # noqa: E501
"Sun Feb 10 00:25:55 2019 Projection: 7 angle: 2.2064 Monitor 3 before: 5635673 Monitor 3 after: 5777316", # noqa: E501
"Sun Feb 10 00:26:29 2019 Projection: 8 angle: 2.5216 Monitor 3 before: 5786535 Monitor 3 after: 5929002", # noqa: E501
"Sun Feb 10 00:27:02 2019 Projection: 9 angle: 2.8368 Monitor 3 before: 5938142 Monitor 3 after: 6078866", # noqa: E501
]
return IMATLogFile(data, "/tmp/fake")


def generate_csv_logfile() -> IMATLogFile:
data = [
CSVLogParser.EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE,
"Sun Feb 10 00:22:04 2019,Projection,0,angle: 0.0,Monitor 3 before: 4577907,Monitor 3 after: 4720271",
"Sun Feb 10 00:22:37 2019,Projection,1,angle: 0.3152,Monitor 3 before: 4729337,Monitor 3 after: 4871319",
"Sun Feb 10 00:23:10 2019,Projection,2,angle: 0.6304,Monitor 3 before: 4879923,Monitor 3 after: 5022689",
"Sun Feb 10 00:23:43 2019,Projection,3,angle: 0.9456,Monitor 3 before: 5031423,Monitor 3 after: 5172216",
"Sun Feb 10 00:24:16 2019,Projection,4,angle: 1.2608,Monitor 3 before: 5180904,Monitor 3 after: 5322691",
"Sun Feb 10 00:24:49 2019,Projection,5,angle: 1.576,Monitor 3 before: 5334225,Monitor 3 after: 5475239",
"Sun Feb 10 00:25:22 2019,Projection,6,angle: 1.8912,Monitor 3 before: 5483964,Monitor 3 after: 5626608",
"Sun Feb 10 00:25:55 2019,Projection,7,angle: 2.2064,Monitor 3 before: 5635673,Monitor 3 after: 5777316",
"Sun Feb 10 00:26:29 2019,Projection,8,angle: 2.5216,Monitor 3 before: 5786535,Monitor 3 after: 5929002",
"Sun Feb 10 00:27:02 2019,Projection,9,angle: 2.8368,Monitor 3 before: 5938142,Monitor 3 after: 6078866",
]
return IMATLogFile(data, "/tmp/fake")
14 changes: 11 additions & 3 deletions mantidimaging/core/data/test/images_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from six import StringIO

from mantidimaging.core.data import Images
from mantidimaging.core.data.test.fake_logfile import generate_logfile
from mantidimaging.core.data.test.fake_logfile import generate_csv_logfile, generate_txt_logfile
from mantidimaging.core.operations.crop_coords import CropCoordinatesFilter
from mantidimaging.core.operation_history import const
from mantidimaging.core.utility.sensible_roi import SensibleROI
Expand Down Expand Up @@ -184,7 +184,15 @@ def test_create_empty_images(self):

def test_get_projection_angles_from_logfile(self):
images = generate_images()
images.log_file = generate_logfile()
images.log_file = generate_txt_logfile()
expected = np.deg2rad(np.asarray([0.0, 0.3152, 0.6304, 0.9456, 1.2608, 1.576, 1.8912, 2.2064, 2.5216, 2.8368]))
actual = images.projection_angles(360.0)
self.assertEqual(len(actual.value), len(expected))
np.testing.assert_equal(actual.value, expected)

def test_get_projection_angles_from_logfile_csv(self):
images = generate_images()
images.log_file = generate_csv_logfile()
expected = np.deg2rad(np.asarray([0.0, 0.3152, 0.6304, 0.9456, 1.2608, 1.576, 1.8912, 2.2064, 2.5216, 2.8368]))
actual = images.projection_angles(360.0)
self.assertEqual(len(actual.value), len(expected))
Expand All @@ -202,7 +210,7 @@ def test_get_projection_angles_no_logfile(self):

def test_metadata_gets_updated_with_logfile(self):
images = generate_images()
images.log_file = generate_logfile()
images.log_file = generate_txt_logfile()
self.assertEqual(images.log_file.source_file, images.metadata[const.LOG_FILE])

def test_set_projection_angles(self):
Expand Down
6 changes: 1 addition & 5 deletions mantidimaging/core/io/loader/loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,8 @@ def read_in_file_information(input_path,


def load_log(log_file: str) -> IMATLogFile:
data = []
with open(log_file, 'r') as f:
for line in f:
data.append(line.strip().split(" "))

return IMATLogFile(data, log_file)
return IMATLogFile(f.readlines(), log_file)


def load_p(parameters: ImageParameters, dtype, progress) -> Images:
Expand Down
5 changes: 5 additions & 0 deletions mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,12 @@ class MonitorNormalisation(BaseFilter):

@staticmethod
def filter_func(images: Images, cores=None, chunksize=None, progress=None) -> Images:
if images.num_projections == 1:
# we can't really compute the preview as the image stack copy
# passed in doesn't have the logfile in it
return images
counts = images.counts()

if counts is None:
raise RuntimeError("No loaded log values for this stack.")

Expand Down
136 changes: 100 additions & 36 deletions mantidimaging/core/utility/imat_log_file_parser.py
View file
Original file line number Diff line number Diff line change
@@ -1,72 +1,138 @@
# Copyright (C) 2020 ISIS Rutherford Appleton Laboratory UKRI
# SPDX - License - Identifier: GPL-3.0-or-later

import csv
from enum import Enum, auto
from itertools import zip_longest
from typing import List, Dict
from typing import Dict, List

import numpy

from mantidimaging.core.utility.data_containers import ProjectionAngles, Counts
from mantidimaging.core.utility.data_containers import Counts, ProjectionAngles

EXPECTED_HEADER_FOR_IMAT_LOG_FILE = [
'TIME STAMP IMAGE TYPE', 'IMAGE COUNTER', 'COUNTS BM3 before image', 'COUNTS BM3 after image'
]

def _get_projection_number(s: str) -> int:
return int(s[s.find(":") + 1:s.find("a")].strip())


def _get_angle(s: str) -> str:
return s[s.rfind(":") + 1:].strip()


class IMATLogColumn(Enum):
TIMESTAMP = auto()
# currently these 2 are the same column because
# the log file formatting is inconsistent
IMAGE_TYPE_IMAGE_COUNTER = auto()
PROJECTION_NUMBER = auto()
PROJECTION_ANGLE = auto()
COUNTS_BEFORE = auto()
COUNTS_AFTER = auto()


class IMATLogFile:
def __init__(self, data: List[List[str]], source_file: str):
self._source_file = source_file
self._data: Dict[IMATLogColumn, List] = {
class TextLogParser:
EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE = \
' TIME STAMP IMAGE TYPE IMAGE COUNTER COUNTS BM3 before image COUNTS BM3 after image\n'

def __init__(self, data: List[str]) -> None:
self.data = [line.strip().split(" ") for line in data]

def parse(self) -> Dict[IMATLogColumn, List]:
parsed_log: Dict[IMATLogColumn, List] = {
IMATLogColumn.TIMESTAMP: [],
IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER: [],
IMATLogColumn.PROJECTION_NUMBER: [],
IMATLogColumn.PROJECTION_ANGLE: [],
IMATLogColumn.COUNTS_BEFORE: [],
IMATLogColumn.COUNTS_AFTER: []
}

if EXPECTED_HEADER_FOR_IMAT_LOG_FILE != data[0]:
raise RuntimeError(
"The Log file found for this dataset does not seem to have the correct header for an IMAT log file.\n"
"The header is expected to contain the names of the columns:\n"
f"{str(EXPECTED_HEADER_FOR_IMAT_LOG_FILE)}")

# ignores the headers (index 0) as they're not the same as the data anyway
# and index 1 is an empty line
for line in data[2:]:
self._data[IMATLogColumn.TIMESTAMP].append(line[0])
self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER].append(line[1])
self._data[IMATLogColumn.COUNTS_BEFORE].append(line[2])
self._data[IMATLogColumn.COUNTS_AFTER].append(line[3])
for line in self.data[2:]:
parsed_log[IMATLogColumn.TIMESTAMP].append(line[0])
parsed_log[IMATLogColumn.PROJECTION_NUMBER].append(_get_projection_number(line[1]))
parsed_log[IMATLogColumn.PROJECTION_ANGLE].append(float(_get_angle(line[1])))
parsed_log[IMATLogColumn.COUNTS_BEFORE].append(int(_get_angle(line[2])))
parsed_log[IMATLogColumn.COUNTS_AFTER].append(int(_get_angle(line[3])))

return parsed_log

@staticmethod
def validate(file_contents) -> bool:
if TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE != file_contents[0]:
return False
return True


class CSVLogParser:
EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE = \
"TIME STAMP,IMAGE TYPE,IMAGE COUNTER,COUNTS BM3 before image,COUNTS BM3 after image\n"

def __init__(self, data: List[str]) -> None:
self.data = data

def parse(self) -> Dict[IMATLogColumn, List]:
parsed_log: Dict[IMATLogColumn, List] = {
IMATLogColumn.TIMESTAMP: [],
IMATLogColumn.PROJECTION_NUMBER: [],
IMATLogColumn.PROJECTION_ANGLE: [],
IMATLogColumn.COUNTS_BEFORE: [],
IMATLogColumn.COUNTS_AFTER: []
}

reader = csv.reader(self.data)

# skip headings
next(reader)

for row in reader:
parsed_log[IMATLogColumn.TIMESTAMP].append(row[0])
parsed_log[IMATLogColumn.PROJECTION_NUMBER].append(int(row[2]))
angle_raw = row[3]
parsed_log[IMATLogColumn.PROJECTION_ANGLE].append(float(_get_angle(angle_raw)))

counts_before_raw = row[4]
parsed_log[IMATLogColumn.COUNTS_BEFORE].append(int(_get_angle(counts_before_raw)))

counts_after_raw = row[5]
parsed_log[IMATLogColumn.COUNTS_AFTER].append(int(_get_angle(counts_after_raw)))

return parsed_log

@staticmethod
def validate(file_contents) -> bool:
if CSVLogParser.EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE != file_contents[0]:
return False
return True


class IMATLogFile:
def __init__(self, data: List[str], source_file: str):
self._source_file = source_file

self.parser = self.find_parser(data)
self._data = self.parser.parse()

@staticmethod
def find_parser(data: List[str]):
if TextLogParser.validate(data):
return TextLogParser(data)
elif CSVLogParser.validate(data):
return CSVLogParser(data)
else:
raise RuntimeError("The format of the log file is not recognised.")

@property
def source_file(self) -> str:
return self._source_file

def projection_numbers(self):
proj_nums = numpy.zeros(len(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]), dtype=numpy.uint32)
for i, angle_str in enumerate(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]):
if "angle:" not in angle_str:
raise ValueError("Projection angles loaded from logfile do not have the correct formatting!")
proj_nums[i] = int(angle_str[angle_str.find(": ") + 1:angle_str.find("a")])

proj_nums = numpy.zeros(len(self._data[IMATLogColumn.PROJECTION_NUMBER]), dtype=numpy.uint32)
proj_nums[:] = self._data[IMATLogColumn.PROJECTION_NUMBER]
return proj_nums

def projection_angles(self) -> ProjectionAngles:
angles = numpy.zeros(len(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]))
for i, angle_str in enumerate(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]):
if "angle:" not in angle_str:
raise ValueError("Projection angles loaded from logfile do not have the correct formatting!")
angles[i] = float(angle_str[angle_str.rfind(": ") + 1:])

angles = numpy.zeros(len(self._data[IMATLogColumn.PROJECTION_ANGLE]))
angles[:] = self._data[IMATLogColumn.PROJECTION_ANGLE]
return ProjectionAngles(numpy.deg2rad(angles))

def counts(self) -> Counts:
Expand All @@ -75,8 +141,6 @@ def counts(self) -> Counts:
after] in enumerate(zip(self._data[IMATLogColumn.COUNTS_BEFORE],
self._data[IMATLogColumn.COUNTS_AFTER])):
# clips the string before the count number
before = before[before.rfind(":") + 1:]
after = after[after.rfind(":") + 1:]
counts[i] = float(after) - float(before)

return Counts(counts)
Expand Down
Loading

0 comments on commit c5c5e9e

Please sign in to comment.