Merge pull request #43 from nasa/4_adapter_parser_refactor

4 adapter parser refactor

onair/config/default_config.ini

@@ -11,5 +11,4 @@ PluginList = {'generic_plugin':'plugins/generic/generic_plugin.py'}
 [RUN_FLAGS]
 IO_Flag = true
 Dev_Flag = false
-SBN_Flag = false
 Viz_Flag = false

onair/data_handling/parsers/csv_parser.py → onair/data_handling/csv_parser.py

@@ -14,41 +14,36 @@
 import os
 import pandas as pd
 
-from .on_air_parser import OnAirParser
-from ...src.util.print_io import *
+from .on_air_data_source import OnAirDataSource
+from ..src.util.print_io import *
 from .parser_util import * 
 
-class CSV(OnAirParser):
+class CSV(OnAirDataSource):
+
     def process_data_file(self, data_file):
-        labels, data = self.parse_csv_data(data_file)
-        self.all_headers = labels
-        self.sim_data = data
+        self.sim_data = self.parse_csv_data(data_file)
+        self.frame_index = 0
 
 ##### INITIAL PROCESSING ####
     def parse_csv_data(self, data_file):
         #Read in the data set
         dataset = pd.read_csv(data_file, delimiter=',', header=0, dtype=str)
         dataset = dataset.loc[:, ~dataset.columns.str.contains('^Unnamed')]
 
-        all_headers = list(dataset.columns.values)
-        #Find the 'Time' header in the list in order to match 42 file formatting 
-        # Converting
-        upperCaseStringHeaders = [x.upper().strip() for x in all_headers if isinstance(x, str)]
-
         #Initialize the entire data dictionary
         all_data = []
         for index, row in dataset.iterrows():
             rowVals = floatify_input(list(row))
             all_data.append(floatify_input(list(row)))
 
-        return all_headers, all_data 
+        return all_data
 
     def parse_meta_data_file(self, meta_data_file, ss_breakdown):
-        parsed_configs = extract_meta_data(meta_data_file)
+        parsed_meta_data = extract_meta_data(meta_data_file)
         if ss_breakdown == False:
-            num_elements = len(parsed_configs['subsystem_assignments'])
-            parsed_configs['subsystem_assignments'] = [['MISSION'] for elem in range(num_elements)]
-        return parsed_configs
+            num_elements = len(parsed_meta_data['subsystem_assignments'])
+            parsed_meta_data['subsystem_assignments'] = [['MISSION'] for elem in range(num_elements)]
+        return parsed_meta_data
 
 ##### GETTERS ##################################
     def get_sim_data(self):
@@ -59,3 +54,12 @@ def get_just_data(self):
 
     def get_vehicle_metadata(self):
         return self.all_headers, self.binning_configs['test_assignments']
+
+    # Get the data at self.index and increment the index
+    def get_next(self):
+        self.frame_index = self.frame_index + 1
+        return self.sim_data[self.frame_index - 1]
+
+    # Return whether or not the index has finished traveling through the data
+    def has_more(self):
+        return self.frame_index < len(self.sim_data)

onair/data_handling/parsers/on_air_parser.py → onair/data_handling/on_air_data_source.py

@@ -10,35 +10,51 @@
 from abc import ABC, abstractmethod
 from .parser_util import * 
 
-class OnAirParser(ABC):
+class OnAirDataSource(ABC):
     def __init__(self, data_file, meta_file, ss_breakdown = False):
         """An initial parsing needs to happen in order to use the parser classes
             This means that, if you want to use this class to parse in real time,
             it needs to at least have seen one sample of the anticipated format """
 
         self.raw_data_file = data_file
         self.meta_data_file = meta_file
-        self.all_headers = {}
+
+        self.all_headers = []
         self.sim_data = {}
         self.binning_configs = {}
 
         configs = self.parse_meta_data_file(self.meta_data_file, ss_breakdown)
         self.binning_configs['subsystem_assignments'] = configs['subsystem_assignments']
         self.binning_configs['test_assignments'] = configs['test_assignments']
         self.binning_configs['description_assignments'] = configs['description_assignments']
+        self.all_headers = configs['data_labels']
 
         self.process_data_file(self.raw_data_file)
 
+    @abstractmethod
+    def parse_meta_data_file(self, meta_data_file, ss_breakdown):
+        """
+        Create the configs that will be used to populate the binning_configs for the data files
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def process_data_file(self, data_file):
         """
-        Adjust each individual data_file before parsing into binning_configs
+        Read data frames from the specified file.
         """
         raise NotImplementedError
 
     @abstractmethod
-    def parse_meta_data_file(self, meta_data_file, ss_breakdown):
+    def get_next(self):
         """
-        Create the configs that will be used to populate the binning_configs for the data files
+        Return a frame of data
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def has_more(self):
+        """
+        Used by file-based data to indicate if there are more frames (True) or if the end of the file has been reached (False)
         """
         raise NotImplementedError

onair/data_handling/parsers/tlm_json_parser.py → onair/data_handling/tlm_json_parser.py

@@ -57,6 +57,7 @@ def parseTlmConfJson(file_path):
     configs['subsystem_assignments'] = subsys_assignments
     configs['test_assignments'] = mnemonic_tests
     configs['description_assignments'] = descriptions
+    configs['data_labels'] = labels
 
     return configs
 

onair/src/run_scripts/execution_engine.py

@@ -30,7 +30,6 @@ def __init__(self, config_file='', run_name='', save_flag=False):
         # Init Flags 
         self.IO_Flag = False
         self.Dev_Flag = False
-        self.SBN_Flag = False
         self.Viz_Flag = False
 
         # Init Paths 
@@ -101,7 +100,6 @@ def parse_configs(self, config_filepath):
         ## Parse Optional Data: Flags
         self.IO_Flag = config['RUN_FLAGS'].getboolean('IO_Flag')
         self.Dev_Flag = config['RUN_FLAGS'].getboolean('Dev_Flag')
-        self.SBN_Flag = config['RUN_FLAGS'].getboolean('SBN_Flag')
         self.Viz_Flag = config['RUN_FLAGS'].getboolean('Viz_Flag')
 
         ## Parse Optional Data: Benchmarks
@@ -113,17 +111,13 @@ def parse_configs(self, config_filepath):
             pass
 
     def parse_data(self, parser_name, parser_file_name, data_file_name, metadata_file_name, subsystems_breakdown=False):
-        parser = importlib.import_module('onair.data_handling.parsers.' + parser_file_name)
+        parser = importlib.import_module('onair.data_handling.' + parser_file_name)
         parser_class = getattr(parser, parser_name) # This could be simplified if the parsers all extend a parser class... but this works for now
-        # TODO: should this us os.path.join?
-        tm_data_path = os.environ['RUN_PATH'] + data_file_name
-        tm_metadata_path = os.environ['RUN_PATH'] +  metadata_file_name
-        # TODO: This will be changed on an OnAIR Data Source
-        data_parser = parser_class(tm_data_path, tm_metadata_path, subsystems_breakdown)
+        data_parser = parser_class(data_file_name, metadata_file_name, subsystems_breakdown)
         self.simDataParser = data_parser
 
     def setup_sim(self):
-        self.sim = Simulator(self.sim_name, self.simDataParser, self.plugin_list, self.SBN_Flag)
+        self.sim = Simulator(self.sim_name, self.simDataParser, self.plugin_list)
         try:
             fls = ast.literal_eval(self.benchmarkFiles)
             fp = os.path.dirname(os.path.realpath(__file__)) + '/../..' + self.benchmarkFilePath