Import data sources in the same ways a plugins

onair/config/default_config.ini

@@ -3,8 +3,7 @@ TelemetryDataFilePath = onair/data/raw_telemetry_data/data_physics_generation/Er
 TelemetryFile = 700_crash_to_earth_1.csv
 TelemetryMetadataFilePath = onair/data/telemetry_configs/
 MetaFile = data_physics_generation_CONFIG.json
-ParserFileName = csv_parser
-ParserName = CSV
+ParserFileName = onair/data_handling/csv_parser.py
 PluginList = {'generic_plugin':'plugins/generic/generic_plugin.py'}
 
 [RUN_FLAGS]

onair/data_handling/csv_parser.py

@@ -14,11 +14,11 @@
 import os
 import pandas as pd
 
-from .on_air_data_source import OnAirDataSource
-from ..src.util.print_io import *
-from .parser_util import * 
+from onair.data_handling.on_air_data_source import OnAirDataSource
+from onair.src.util.print_io import *
+from onair.data_handling.parser_util import *
 
-class CSV(OnAirDataSource):
+class DataSource(OnAirDataSource):
 
     def process_data_file(self, data_file):
         self.sim_data = self.parse_csv_data(data_file)

onair/src/run_scripts/execution_engine.py

@@ -45,7 +45,6 @@ def __init__(self, config_file='', run_name='', save_flag=False):
 
         # Init parsing/sim info
         self.parser_file_name = ''
-        self.parser_name = ''
         self.simDataParser = None
         self.sim = None
 
@@ -58,7 +57,7 @@ def __init__(self, config_file='', run_name='', save_flag=False):
         if config_file != '':
             self.init_save_paths()
             self.parse_configs(config_file)
-            self.parse_data(self.parser_name, self.parser_file_name, self.fullTelemetryFileName, self.fullMetaDataFileName)
+            self.parse_data(self.parser_file_name, self.fullTelemetryFileName, self.fullMetaDataFileName)
             self.setup_sim()
 
     def parse_configs(self, config_filepath):
@@ -78,7 +77,6 @@ def parse_configs(self, config_filepath):
 
             ## Parse Required Data: Names
             self.parser_file_name = config['DEFAULT']['ParserFileName']
-            self.parser_name = config['DEFAULT']['ParserName']
 
             ## Parse Required Data: Plugin name to path dict
             config_plugin_list = config['DEFAULT']['PluginList']
@@ -108,11 +106,11 @@ def parse_configs(self, config_filepath):
         except:
             pass
 
-    def parse_data(self, parser_name, parser_file_name, data_file_name, metadata_file_name, subsystems_breakdown=False):
-        parser = importlib.import_module('onair.data_handling.' + parser_file_name)
-        parser_class = getattr(parser, parser_name) # This could be simplified if the parsers all extend a parser class... but this works for now
-        data_parser = parser_class(data_file_name, metadata_file_name, subsystems_breakdown)
-        self.simDataParser = data_parser
+    def parse_data(self, parser_file_name, data_file_name, metadata_file_name, subsystems_breakdown=False):
+        data_source_spec = importlib.util.spec_from_file_location('data_source', parser_file_name)
+        data_source_module = importlib.util.module_from_spec(data_source_spec)
+        data_source_spec.loader.exec_module(data_source_module)
+        self.simDataParser = data_source_module.DataSource(data_file_name, metadata_file_name, subsystems_breakdown)
 
     def setup_sim(self):
         self.sim = Simulator(self.simDataParser, self.plugin_list)

test/onair/data_handling/test_csv_parser.py

@@ -12,11 +12,11 @@
 from mock import MagicMock
 
 import onair.data_handling.csv_parser as csv_parser
-from onair.data_handling.csv_parser import CSV
+from onair.data_handling.csv_parser import DataSource
 
 @pytest.fixture
 def setup_teardown():
-    pytest.cut = CSV.__new__(CSV)
+    pytest.cut = DataSource.__new__(DataSource)
     yield 'setup_teardown'
 
 # process_data_per_data_file tests

test/onair/src/run_scripts/test_execution_engine.py

@@ -46,7 +46,6 @@ def test_ExecutionEngine__init__sets_expected_values_but_does_no_calls_when_conf
     assert cut.benchmarkFiles == ''
     assert cut.benchmarkIndices == ''
     assert cut.parser_file_name == ''
-    assert cut.parser_name == ''
     assert cut.simDataParser == None
     assert cut.sim == None
     assert cut.save_flag == arg_save_flag
@@ -77,7 +76,7 @@ def test_ExecutionEngine__init__does_calls_when_config_file_is_an_occupied_strin
     assert cut.parse_configs.call_count == 1
     assert cut.parse_configs.call_args_list[0].args == (arg_config_file, )
     assert cut.parse_data.call_count == 1
-    assert cut.parse_data.call_args_list[0].args == (cut.parser_name, cut.parser_file_name, cut.dataFilePath, cut.metadataFilePath, )
+    assert cut.parse_data.call_args_list[0].args == (cut.parser_file_name, cut.dataFilePath, cut.metadataFilePath, )
     assert cut.setup_sim.call_count == 1
 
 def test_ExecutionEngine__init__accepts_no_arguments_using_defaults_instead_with_config_file_default_as_empty_string(mocker):
@@ -131,7 +130,6 @@ def test_ExecutionEngine_parse_configs_raises_KeyError_with_config_file_info_whe
                     'BenchmarkFiles':MagicMock(),
                     'BenchmarkIndices':MagicMock(),
                     'ParserFileName':MagicMock(),
-                    'ParserName':MagicMock(),
                     'PluginList':MagicMock()
                     }
     required_keys = [item for item in list(fake_default.keys()) if 'Benchmark' not in item]
@@ -166,7 +164,7 @@ def test_ExecutionEngine_parse_configs_raises_ValueError_when_PluginList_from_co
     fake_plugin_list.body = MagicMock()
     fake_default_item = MagicMock()
     fake_config.__getitem__.return_value = fake_default_item
-    fake_default_item.__getitem__.side_effect = [fake_paths_and_filenames] * 4 + [None] * 2 + [fake_plugin_list]
+    fake_default_item.__getitem__.side_effect = [fake_paths_and_filenames] * 4 + [None] + [fake_plugin_list]
     fake_config_read_result = MagicMock()
     fake_config_read_result.__len__.return_value = 1
 
@@ -200,7 +198,7 @@ def test_ExecutionEngine_parse_configs_raises_ValueError_when_PluginList_from_co
     fake_plugin_list.body.keys.__len__.return_value = 0
     fake_default_item = MagicMock()
     fake_config.__getitem__.return_value = fake_default_item
-    fake_default_item.__getitem__.side_effect = [fake_paths_and_filenames] * 4 + [None] * 2 + [fake_plugin_list]
+    fake_default_item.__getitem__.side_effect = [fake_paths_and_filenames] * 4 + [None] + [fake_plugin_list]
     fake_config_read_result = MagicMock()
     fake_config_read_result.__len__.return_value = 1
 
@@ -238,7 +236,7 @@ def test_ExecutionEngine_parse_configs_raises_FileNotFoundError_when_given_plugi
     fake_temp_iter = iter([fake_plugin_name])
     fake_default_item = MagicMock()
     fake_config.__getitem__.return_value = fake_default_item
-    fake_default_item.__getitem__.side_effect = [fake_paths_and_filenames] * 4 + [None] * 2 + [fake_plugin_list]
+    fake_default_item.__getitem__.side_effect = [fake_paths_and_filenames] * 4 + [None] + [fake_plugin_list]
     fake_config_read_result = MagicMock()
     fake_config_read_result.__len__.return_value = 1
 
@@ -276,7 +274,6 @@ def test_ExecutionEngine_parse_configs_sets_all_items_without_error(mocker):
                     'BenchmarkFiles':MagicMock(),
                     'BenchmarkIndices':MagicMock(),
                     'ParserFileName':MagicMock(),
-                    'ParserName':MagicMock(),
                     'PluginList':"{fake_name:fake_path}"
                     }
     fake_run_flags = MagicMock()
@@ -330,7 +327,6 @@ def test_ExecutionEngine_parse_configs_sets_all_items_without_error(mocker):
     assert cut.benchmarkFiles == fake_default['BenchmarkFiles']
     assert cut.benchmarkIndices == fake_default['BenchmarkIndices']
     assert cut.parser_file_name == fake_default['ParserFileName']
-    assert cut.parser_name == fake_default['ParserName']
     assert cut.plugin_list == fake_temp_plugin_list
     assert fake_run_flags.getboolean.call_count == 3
     assert fake_run_flags.getboolean.call_args_list[0].args == ('IO_Flag', )
@@ -350,7 +346,6 @@ def test_ExecutionEngine_parse_configs_bypasses_benchmarks_when_access_raises_er
                     'TelemetryMetadataFilePath':MagicMock(),
                     'MetaFile':MagicMock(),
                     'ParserFileName':MagicMock(),
-                    'ParserName':MagicMock(),
                     'PluginList':"{fake_name:fake_path}"
                     }
     fake_run_flags = MagicMock()
@@ -401,7 +396,6 @@ def test_ExecutionEngine_parse_configs_raises_KeyError_with_config_file_info_whe
                     'BenchmarkFiles':MagicMock(),
                     'BenchmarkIndices':MagicMock(),
                     'ParserFileName':MagicMock(),
-                    'ParserName':MagicMock(),
                     }
     required_keys = [item for item in list(fake_default.keys()) if 'Benchmark' not in item]
     missing_key = pytest.gen.choice(required_keys)
@@ -428,46 +422,37 @@ def test_ExecutionEngine_parse_configs_raises_KeyError_with_config_file_info_whe
 # parse_data tests
 def test_ExecutionEngine_parse_data_sets_the_simDataParser_to_the_data_parser(mocker):
     # Arrange
-    arg_parser_name = MagicMock()
     arg_parser_file_name = MagicMock()
     arg_dataFile = str(MagicMock())
     arg_metadataFile = str(MagicMock())
     arg_subsystems_breakdown = MagicMock()
 
-    class FakeParser:
-        def __init__(self, data_file, metadata_file, subsystems_breakdown):
-            pass
-
-    fake_parser = MagicMock()
-    fake_parser_class = FakeParser
+    fake_spec = MagicMock()
+    fake_module = MagicMock()
     fake_parser_class_instance = MagicMock()
-    fake_parsed_data = MagicMock()
 
     cut = ExecutionEngine.__new__(ExecutionEngine)
-    cut.telemetryFile = MagicMock()
-    cut.metaFile = MagicMock()
 
-    mocker.patch(execution_engine.__name__ + '.importlib.import_module', return_value=fake_parser)
-    mocker.patch(execution_engine.__name__ + '.getattr', return_value=fake_parser_class)
-    mocker.patch.object(fake_parser_class, '__new__', return_value=fake_parser_class_instance)
+    mocker.patch(execution_engine.__name__ + '.importlib.util.spec_from_file_location', return_value=fake_spec)
+    mocker.patch(execution_engine.__name__ + '.importlib.util.module_from_spec', return_value=fake_module)
+    mocker.patch.object(fake_spec, 'loader.exec_module', return_value=None)
+    mocker.patch.object(fake_module, 'DataSource', return_value=fake_parser_class_instance)
 
     # Act
-    cut.parse_data(arg_parser_name, arg_parser_file_name, arg_dataFile, arg_metadataFile, arg_subsystems_breakdown)
+    cut.parse_data(arg_parser_file_name, arg_dataFile, arg_metadataFile, arg_subsystems_breakdown)
 
     # Assert
-    assert execution_engine.importlib.import_module.call_count == 1
-    assert execution_engine.importlib.import_module.call_args_list[0].args == ('data_handling.parsers.' + arg_parser_file_name, )
-    assert execution_engine.getattr.call_count == 1
-    assert execution_engine.getattr.call_args_list[0].args == (fake_parser, arg_parser_name,)
+    assert execution_engine.importlib.util.spec_from_file_location.call_count == 1
+    assert execution_engine.importlib.util.spec_from_file_location.call_args_list[0].args == ('data_source', arg_parser_file_name, )
+    assert execution_engine.importlib.util.module_from_spec.call_count == 1
+    assert execution_engine.importlib.util.module_from_spec.call_args_list[0].args == (fake_spec, )
+    assert fake_spec.loader.exec_module.call_count == 1
+    assert fake_module.DataSource.call_count == 1
+    assert fake_module.DataSource.call_args_list[0].args == (arg_dataFile, arg_metadataFile, arg_subsystems_breakdown, )
     assert cut.simDataParser == fake_parser_class_instance
-    assert fake_parser_class.__new__.call_count == 1
-    assert fake_parser_class.__new__.call_args_list[0].args == (fake_parser_class, arg_dataFile, arg_metadataFile, arg_subsystems_breakdown, )
-
-    # subsystems_breakdown
 
 def test_ExecutionEngine_parse_data_argument_subsystems_breakdown_optional_default_is_False(mocker):
     # Arrange
-    arg_parser_name = MagicMock()
     arg_parser_file_name = MagicMock()
     arg_dataFile = MagicMock()
     arg_metadataFile = str(MagicMock())
@@ -482,26 +467,23 @@ def __init__(self, data_file, meta_file, subsystems_breakdown):
             FakeParser.init_meta_data_file = meta_file
             FakeParser.init_subsystems_breakdown = subsystems_breakdown
 
-    fake_parser = MagicMock()
-    fake_parser_class = FakeParser
-    fake_run_path = str(MagicMock())
-    fake_environ = {'RUN_PATH':fake_run_path}
-    fake_parsed_data = MagicMock()
-    fake_processdSimData = MagicMock()
+    fake_spec = MagicMock()
+    fake_module = MagicMock()
+    fake_parser_class_instance = MagicMock()
 
     cut = ExecutionEngine.__new__(ExecutionEngine)
-    cut.telemetryFile = MagicMock()
-    cut.metaFile = MagicMock()
 
-    mocker.patch(execution_engine.__name__ + '.importlib.import_module', return_value=fake_parser)
-    mocker.patch(execution_engine.__name__ + '.getattr', return_value=fake_parser_class)
-    mocker.patch.dict(execution_engine.__name__ + '.os.environ', fake_environ)
+    mocker.patch(execution_engine.__name__ + '.importlib.util.spec_from_file_location', return_value=fake_spec)
+    mocker.patch(execution_engine.__name__ + '.importlib.util.module_from_spec', return_value=fake_module)
+    mocker.patch.object(fake_spec, '.loader.exec_module', return_value=None)
+    mocker.patch.object(fake_module, '.DataSource', return_value=fake_parser_class_instance)
 
     # Act
-    cut.parse_data(arg_parser_name, arg_parser_file_name, arg_dataFile, arg_metadataFile)
+    cut.parse_data(arg_parser_file_name, arg_dataFile, arg_metadataFile)
 
     # Assert
-    assert FakeParser.init_subsystems_breakdown == False
+    assert fake_module.DataSource.call_count == 1
+    assert fake_module.DataSource.call_args_list[0].args == (arg_dataFile, arg_metadataFile, False, )
 
 # setup_sim tests
 def test_ExecutionEngine_setup_sim_sets_self_sim_to_new_Simulator_and_sets_benchmark_data_when_no_exceptions_are_encountered(mocker):