diff --git a/onair/data_handling/parsers/csv_parser.py b/onair/data_handling/parsers/csv_parser.py index 9ea27558..84fafb86 100644 --- a/onair/data_handling/parsers/csv_parser.py +++ b/onair/data_handling/parsers/csv_parser.py @@ -68,7 +68,7 @@ def parse_csv_data(self, dataFile): return all_headers, all_data def parse_config_data(self, configFile, ss_breakdown): - parsed_configs = extract_configs(self.metadata_filepath, configFile, csv=True) + parsed_configs = extract_configs(self.metadata_filepath, configFile) if ss_breakdown == False: num_elements = len(parsed_configs['subsystem_assignments']) parsed_configs['subsystem_assignments'] = [['MISSION'] for elem in range(num_elements)] diff --git a/onair/data_handling/parsers/parser_util.py b/onair/data_handling/parsers/parser_util.py index 646e9ffa..dedc970f 100644 --- a/onair/data_handling/parsers/parser_util.py +++ b/onair/data_handling/parsers/parser_util.py @@ -13,7 +13,7 @@ import datetime ## Method to extract configuration data and return 3 dictionaries -def extract_configs(configFilePath, configFile, csv = False): +def extract_configs(configFilePath, configFile): assert configFile != '' configs = parseTlmConfJson(configFilePath + configFile) @@ -35,19 +35,6 @@ def extract_configs(configFilePath, configFile, csv = False): test_assign[j] = test + limits return configs - -def process_filepath(path, return_config=False, csv = False): - if csv: - filename = path.split(os.sep)[-1].replace('_CONFIG', '') - filename = filename.replace('.txt', '.csv') - if return_config == True: - filename = filename.replace('.csv', '_CONFIG.txt') - return filename - else: - filename = path.split(os.sep)[-1].replace('_CONFIG', '') - if return_config == True: - filename = filename.replace('.txt', '_CONFIG.txt') - return filename def floatify_input(_input, remove_str=False): floatified = [] diff --git a/onair/src/data_driven_components/data_driven_learning.py b/onair/src/data_driven_components/data_driven_learning.py index 07287659..4a63ad00 100644 --- a/onair/src/data_driven_components/data_driven_learning.py +++ b/onair/src/data_driven_components/data_driven_learning.py @@ -11,6 +11,7 @@ Data driven learning class for managing all data driven AI components """ import importlib.util +import importlib.util from ..util.data_conversion import * diff --git a/test/onair/data_handling/parsers/test_csv_parser.py b/test/onair/data_handling/parsers/test_csv_parser.py index a8e9599c..9a3e5712 100644 --- a/test/onair/data_handling/parsers/test_csv_parser.py +++ b/test/onair/data_handling/parsers/test_csv_parser.py @@ -372,7 +372,6 @@ def test_CSV_parse_config_data_returns_call_to_extract_configs_given_metadata_fi # Assert assert csv_parser.extract_configs.call_count == 1 assert csv_parser.extract_configs.call_args_list[0].args == (fake_metadata_filepath, arg_configFile) - assert csv_parser.extract_configs.call_args_list[0].kwargs == {'csv': True} assert csv_parser.len.call_count == 0 assert result == expected_result @@ -400,7 +399,6 @@ def test_CSV_parse_config_data_returns_call_to_extract_configs_given_metadata_fi # Assert assert csv_parser.extract_configs.call_count == 1 assert csv_parser.extract_configs.call_args_list[0].args == (fake_metadata_filepath, arg_configFile) - assert csv_parser.extract_configs.call_args_list[0].kwargs == {'csv': True} assert csv_parser.len.call_count == 1 assert csv_parser.len.call_args_list[0].args == (fake_empty_processed_filepath, ) assert result['subsystem_assignments'] == expected_result @@ -435,7 +433,6 @@ def test_CSV_parse_config_data_returns_call_to_extract_configs_given_metadata_fi # Assert assert csv_parser.extract_configs.call_count == 1 assert csv_parser.extract_configs.call_args_list[0].args == (fake_metadata_filepath, arg_configFile) - assert csv_parser.extract_configs.call_args_list[0].kwargs == {'csv': True} assert csv_parser.len.call_count == 1 assert csv_parser.len.call_args_list[0].args == (fake_processed_filepath, ) assert result['subsystem_assignments'] == expected_result diff --git a/test/onair/data_handling/parsers/test_forty_two_parser.py b/test/onair/data_handling/parsers/test_forty_two_parser.py index f333d155..bf729b78 100644 --- a/test/onair/data_handling/parsers/test_forty_two_parser.py +++ b/test/onair/data_handling/parsers/test_forty_two_parser.py @@ -252,7 +252,6 @@ def test_FortyTwo_parse_config_data_returns_expected_result_when_ss_breakdown_is arg_config_file = MagicMock() fake_metadata_filepath = MagicMock() - fake_filename = MagicMock() fake_subsystem_assignments = [MagicMock()] fake_tests = MagicMock() fake_descs = MagicMock() @@ -264,7 +263,6 @@ def test_FortyTwo_parse_config_data_returns_expected_result_when_ss_breakdown_is 'description_assignments' : fake_descs} mocker.patch(forty_two_parser.__name__ + '.extract_configs', return_value=forced_return_extract_configs) - mocker.patch(forty_two_parser.__name__ + '.process_filepath', return_value=fake_filename) expected_result = { 'subsystem_assignments' : [['MISSION']], 'test_assignments' : fake_tests, @@ -283,7 +281,6 @@ def test_FortyTwo_parse_config_data_returns_expected_result_when_ss_breakdown_is arg_config_file = MagicMock() fake_metadata_filepath = MagicMock() - fake_filename = MagicMock() num_ss_assignments = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10 fake_subsystem_assignments = [MagicMock()] * num_ss_assignments fake_tests = MagicMock() @@ -294,10 +291,8 @@ def test_FortyTwo_parse_config_data_returns_expected_result_when_ss_breakdown_is forced_return_extract_configs = { 'subsystem_assignments' : fake_subsystem_assignments, 'test_assignments' : fake_tests, 'description_assignments' : fake_descs} - forced_return_process_filepath = fake_filename mocker.patch(forty_two_parser.__name__ + '.extract_configs', return_value=forced_return_extract_configs) - mocker.patch(forty_two_parser.__name__ + '.process_filepath', return_value=forced_return_process_filepath) expected_result = { 'subsystem_assignments' : [['MISSION']] * num_ss_assignments, 'test_assignments' : fake_tests, diff --git a/test/onair/data_handling/parsers/test_parser_util.py b/test/onair/data_handling/parsers/test_parser_util.py index 4b455c09..da38ffb6 100644 --- a/test/onair/data_handling/parsers/test_parser_util.py +++ b/test/onair/data_handling/parsers/test_parser_util.py @@ -18,11 +18,10 @@ def test_parser_util_extract_configs_raises_error_when_given_blank_configFile(): # Arrange arg_configFilePath = MagicMock() arg_configFile = '' - arg_csv = MagicMock() # Act with pytest.raises(AssertionError) as e_info: - result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv) + result = parser_util.extract_configs(arg_configFilePath, arg_configFile) # Assert assert e_info.match('') @@ -31,7 +30,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configs_le # Arrange arg_configFilePath = MagicMock() arg_configFile = MagicMock() - arg_csv = MagicMock() fake_subsystem_assignments = MagicMock() fake_tests = MagicMock() @@ -47,7 +45,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configs_le mocker.patch(parser_util.__name__ + '.str2lst') # Act - result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv) + result = parser_util.extract_configs(arg_configFilePath, arg_configFile) # Assert assert parser_util.parseTlmConfJson.call_count == 1 @@ -61,7 +59,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configs_le # Arrange arg_configFilePath = MagicMock() arg_configFile = MagicMock() - arg_csv = MagicMock() fake_subsystem_assignments = [MagicMock()] fake_test_assign = MagicMock() @@ -82,7 +79,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configs_le expected_result['description_assignments'] = fake_descs.copy() # Act - result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv) + result = parser_util.extract_configs(arg_configFilePath, arg_configFile) # Assert assert parser_util.parseTlmConfJson.call_count == 1 @@ -94,7 +91,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config # Arrange arg_configFilePath = MagicMock() arg_configFile = MagicMock() - arg_csv = MagicMock() len_configs = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10 (0 and 1 have own tests) fake_subsystem_assignments = [MagicMock()] * len_configs @@ -116,7 +112,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config expected_result['description_assignments'] = fake_descs.copy() # Act - result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv) + result = parser_util.extract_configs(arg_configFilePath, arg_configFile) # Assert assert parser_util.parseTlmConfJson.call_count == 1 @@ -128,7 +124,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config # Arrange arg_configFilePath = MagicMock() arg_configFile = MagicMock() - arg_csv = MagicMock() len_configs = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10 (0 and 1 have own tests) num_noops = pytest.gen.randint(2, 10) @@ -153,7 +148,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config expected_result['description_assignments'] = fake_descs.copy() # Act - result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv) + result = parser_util.extract_configs(arg_configFilePath, arg_configFile) # Assert assert parser_util.parseTlmConfJson.call_count == 1 @@ -165,7 +160,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config # Arrange arg_configFilePath = MagicMock() arg_configFile = MagicMock() - arg_csv = MagicMock() len_configs = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10 (0 and 1 have own tests) fake_subsystem_assignments = [MagicMock()] * len_configs @@ -201,7 +195,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config expected_result['test_assignments'].append(expected_test_assign) # Act - result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv) + result = parser_util.extract_configs(arg_configFilePath, arg_configFile) # Assert assert parser_util.parseTlmConfJson.call_count == 1 @@ -215,7 +209,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configFile # Arrange arg_configFilePath = MagicMock() arg_configFile = MagicMock() - arg_csv = MagicMock() len_configs = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10 (0 and 1 have own tests) fake_subsystem_assignments = [] @@ -254,137 +247,13 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configFile expected_result['description_assignments'] = fake_descs.copy() # Act - result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv) + result = parser_util.extract_configs(arg_configFilePath, arg_configFile) # Assert assert parser_util.parseTlmConfJson.call_count == 1 assert parser_util.parseTlmConfJson.call_args_list[0].args == (arg_configFilePath + arg_configFile, ) assert result == expected_result -# process_filepath -def test_parser_util_process_filepath_returns_filename_from_path_with_txt_replaced_by_csv_when_given_csv_resolves_to_True_and_given_return_config_is_not_True(mocker): - # Arrange - fake_filename = str(MagicMock()) - fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\' - - arg_path = "" - arg_return_config = False if pytest.gen.randint(0, 1) == 1 else 0 - arg_csv = True if pytest.gen.randint(0, 1) == 1 else MagicMock() - - for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename - arg_path += str(MagicMock()) + fake_os_sep - arg_path += fake_filename + '.txt' - - mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep) - - # Act - result = parser_util.process_filepath(arg_path, arg_return_config, arg_csv) - - # Assert - assert result == fake_filename + '.csv' - -def test_parser_util_process_filepath_returns_filename_from_path_with_txt_replaced_by__CONFIG_dot_txt_when_given_csv_resolves_to_True_and_given_return_config_is_True(mocker): - # Arrange - fake_filename = str(MagicMock()) - fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\' - - arg_path = "" - arg_return_config = True - arg_csv = True if pytest.gen.randint(0, 1) == 1 else MagicMock() - - for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename - arg_path += str(MagicMock()) + fake_os_sep - arg_path += fake_filename + '.txt' - - mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep) - - # Act - result = parser_util.process_filepath(arg_path, arg_return_config, arg_csv) - - # Assert - assert result == fake_filename + '_CONFIG.txt' - -def test_parser_util_process_filepath_returns_filename_from_path_when_given_csv_resolves_to_False_and_given_return_config_is_not_True(mocker): - # Arrange - fake_filename = str(MagicMock()) - fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\' - - arg_path = "" - arg_return_config = False if pytest.gen.randint(0, 1) == 1 else 0 - arg_csv = False if pytest.gen.randint(0, 1) == 1 else 0 - - for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename - arg_path += str(MagicMock()) + fake_os_sep - arg_path += fake_filename + '.txt' - - mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep) - - # Act - result = parser_util.process_filepath(arg_path, arg_return_config, arg_csv) - - # Assert - assert result == fake_filename + '.txt' - -def test_parser_util_process_filepath_returns_filename_from_path_when_given_csv_resolves_to_False_and_given_return_config_is_True(mocker): - # Arrange - fake_filename = str(MagicMock()) - fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\' - - arg_path = "" - arg_return_config = True - arg_csv = False if pytest.gen.randint(0, 1) == 1 else 0 - - for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename - arg_path += str(MagicMock()) + fake_os_sep - arg_path += fake_filename + '.txt' - - mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep) - - # Act - result = parser_util.process_filepath(arg_path, arg_return_config, arg_csv) - - # Assert - assert result == fake_filename + '_CONFIG.txt' - -def test_parser_util_process_filepath_default_given_csv_is_False(mocker): - # Arrange - fake_filename = str(MagicMock()) - fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\' - - arg_path = "" - arg_return_config = True - - for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename - arg_path += str(MagicMock()) + fake_os_sep - arg_path += fake_filename + '.txt' - - mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep) - - # Act - result = parser_util.process_filepath(arg_path, arg_return_config) - - # Assert - assert result == fake_filename + '_CONFIG.txt' - -def test_parser_util_process_filepath_default_given_return_config_is_False(mocker): - # Arrange - fake_filename = str(MagicMock()) - fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\' - - arg_path = "" - - for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename - arg_path += str(MagicMock()) + fake_os_sep - arg_path += fake_filename + '.txt' - - mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep) - - # Act - result = parser_util.process_filepath(arg_path) - - # Assert - assert result == fake_filename + '.txt' - # floatify_input tests def test_parser_util_flotify_input_returns_empty_list_when_given__input_is_vacant(mocker): # Arrange