From 6fb1a1ae8353f6020f979f4acc8b632ff3f9dbbf Mon Sep 17 00:00:00 2001 From: Dylan Chanter Date: Thu, 14 Oct 2021 23:05:16 +0800 Subject: [PATCH 01/19] Inital Commit test case 4 part 1 --- test/integration/test_main.py | 76 +++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index b3f7e7c..c0cebe7 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -10,6 +10,81 @@ # Assume we are in project root directory. TEST_DATA_PATH = Path('./test/input_data/integration') +def test_54_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: + # Uses observation 1294797712, with signal data starting at 1294797712, + # signal with 54 as the input + + working_dir = working_dir / '54_signal_identity_ipfb' + working_dir.mkdir(exist_ok=False, parents=True) + + inv_polyphase_filter_path = working_dir / 'inverse_polyphase_filter.bin' + inv_polyphase_filter = numpy.zeros((13, 256), dtype=numpy.float32) + inv_polyphase_filter[:, -1] = 1 # Identity + write_inv_polyphase_filter(inv_polyphase_filter_path, inv_polyphase_filter) + + input_dir = working_dir / 'input_data' + input_dir.mkdir(exist_ok=False, parents=True) + shutil.copyfile(TEST_DATA_PATH / '1294797712.metafits', input_dir / '1294797712.metafits') + input_file_metadata_template = \ + "HDR_SIZE 4096\nPOPULATED 1\nOBS_ID 1294797712\nSUBOBS_ID 1294797712\nMODE VOLTAGE_START\n" \ + "UTC_START 2021-01-16-02:01:34\nOBS_OFFSET 0\nNBIT 8\nNPOL 2\nNTIMESAMPLES 64000\nNINPUTS 256\n" \ + "NINPUTS_XGPU 256\nAPPLY_PATH_WEIGHTS 0\nAPPLY_PATH_DELAYS 0\nINT_TIME_MSEC 500\nFSCRUNCH_FACTOR 50\n" \ + "APPLY_VIS_WEIGHTS 0\nTRANSFER_SIZE 5275648000\nPROJ_ID G0034\nEXPOSURE_SECS 304\nCOARSE_CHANNEL 113\n" \ + "CORR_COARSE_CHANNEL {}\nSECS_PER_SUBOBS 8\nUNIXTIME 1610762494\nUNIXTIME_MSEC 0\nFINE_CHAN_WIDTH_HZ 10000\n" \ + "NFINE_CHAN 128\nBANDWIDTH_HZ 1280000\nSAMPLE_RATE 1280000\nMC_IP 0.0.0.0\nMC_PORT 0\nMC_SRC_IP 0.0.0.0\n" + + metadata = input_file_metadata_template.encode('ascii') + metadata_padding = bytes(4096 - len(metadata)) + data_block = numpy.full((64000*256,2),(54,0),dtype=numpy.uint8) + print(data_block) + data_bytes_block = data_block.tobytes() + zero_block = bytes(256 * 64000 * 2) + file_path = input_dir / f'1294797712_1294797712_113.sub' + with open(file_path, 'wb') as file: + file.write(metadata) + file.write(metadata_padding) + file.write(zero_block) + for _ in range(160): + file.write(data_bytes_block) + + output_dir = working_dir / 'output_dir' + output_dir.mkdir(exist_ok=False, parents=True) + + result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'false') + + assert result.returncode == 0 + + # Note: these tiles are flagged as faulty and will be skipped: 102, 115, 151, 164, 999, 2013, 2017, 2044, 2047 + tiles = [ + 51, 52, 53, 54, 55, 56, 57, 58, 71, 72, 73, 74, 75, 76, 77, 78, 101, 104, 105, + 106, 107, 108, 111, 112, 113, 114, 116, 117, 118, 121, 122, 123, 124, 125, 126, + 127, 128, 131, 132, 133, 134, 135, 136, 137, 138, 141, 142, 143, 144, 145, 146, 147, + 148, 152, 153, 154, 155, 156, 157, 158, 161, 162, 163, 165, 166, 167, 168, + 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, + 2015, 2016, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, + 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, + 2045, 2046, 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056 + ] + tile_output_filenames = [] + for tile in tiles: + tile_output_filenames.append(f'1294797712_1294797712_{tile}_X.bin') + tile_output_filenames.append(f'1294797712_1294797712_{tile}_Y.bin') + + log_file_name = '1294797712_1294797712_outputlog.txt' + + # Check there's output for all the tiles and polarisations, plus the log file. + output_dir_contents = [entry.name for entry in output_dir.iterdir()] + assert set(output_dir_contents) == set(tile_output_filenames + [log_file_name]) + + # Note: need to manually inspect output log file. + + for filename in tile_output_filenames: + signal = read_output_signal(output_dir / filename) + assert signal.min() == 0 and signal.max() == 0 + # Check that the output downsampling is as expected. The new sampling frequency is 54, manually calculated. + assert len(signal) == 160 * 64000 * 54 + del signal + def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: # Test of valid input data, with all zeros signal data and the identity inverse polyphase filter. @@ -88,3 +163,4 @@ def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: # Check that the output downsampling is as expected. The new sampling frequency is 54, manually calculated. assert len(signal) == 160 * 64000 * 54 del signal # Really don't want big array hanging around + From 24228ce908c475318624ba92311e74d70dc7f03a Mon Sep 17 00:00:00 2001 From: Dylan Chanter Date: Fri, 15 Oct 2021 00:25:53 +0800 Subject: [PATCH 02/19] Test case 4 part 1 --- test/integration/test_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index c0cebe7..b3e5abd 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -85,7 +85,6 @@ def test_54_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: assert len(signal) == 160 * 64000 * 54 del signal - def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: # Test of valid input data, with all zeros signal data and the identity inverse polyphase filter. # Expected output signal is all zeros. From 41c9d685656f0812bcfa220a3f644314c7bd6a42 Mon Sep 17 00:00:00 2001 From: Benjamin Bowers Date: Fri, 15 Oct 2021 09:20:23 +0800 Subject: [PATCH 03/19] Calculating Time domain --- test/integration/test_main.py | 79 ++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index b3f7e7c..788ffbc 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -53,7 +53,7 @@ def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: output_dir.mkdir(exist_ok=False, parents=True) result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'false') - + assert result.returncode == 0 # Note: these tiles are flagged as faulty and will be skipped: 102, 115, 151, 164, 999, 2013, 2017, 2044, 2047 @@ -88,3 +88,80 @@ def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: # Check that the output downsampling is as expected. The new sampling frequency is 54, manually calculated. assert len(signal) == 160 * 64000 * 54 del signal # Really don't want big array hanging around + +def test_complex_pfb(run_script: Path, working_dir: Path) -> None: + # Test of valid input data, with all ones signal data and the polyphase filter is to be all zeros. + working_dir = working_dir / 'one_signal_complex_pfb' + working_dir.mkdir(exist_ok=False, parents=True) + + inv_polyphase_filter_path = working_dir / 'inverse_polyphase_filter_5.bin' + inv_polyphase_filter = numpy.zeros((25, 256), dtype=numpy.float32) + filterVals = [0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1 ,2 ,3 ,4 ,0 ,1 ,2 ,3 ,4, 0, 1, 2, 3, 4] + inv_polyphase_filter = numpy.insert(inv_polyphase_filter, 160, filterVals, axis=1) + input_dir = working_dir / 'input_data' + inverse_polyphase_filter_file = working_dir / 'inverse_polyphase_filter_2.bin' + input_dir.mkdir(exist_ok=False, parents=True) + shutil.copyfile(TEST_DATA_PATH / '1294797712.metafits', input_dir / '1294797712.metafits') + input_file_metadata = \ + "HDR_SIZE 4096\nPOPULATED 1\nOBS_ID 1294797712\nSUBOBS_ID 1294797712\nMODE VOLTAGE_START\n" \ + "UTC_START 2021-01-16-02:01:34\nOBS_OFFSET 0\nNBIT 8\nNPOL 2\nNTIMESAMPLES 64000\nNINPUTS 256\n" \ + "NINPUTS_XGPU 256\nAPPLY_PATH_WEIGHTS 0\nAPPLY_PATH_DELAYS 0\nINT_TIME_MSEC 500\nFSCRUNCH_FACTOR 50\n" \ + "APPLY_VIS_WEIGHTS 0\nTRANSFER_SIZE 5275648000\nPROJ_ID G0034\nEXPOSURE_SECS 304\nCOARSE_CHANNEL {}\n" \ + "CORR_COARSE_CHANNEL {}\nSECS_PER_SUBOBS 8\nUNIXTIME 1610762494\nUNIXTIME_MSEC 0\nFINE_CHAN_WIDTH_HZ 10000\n" \ + "NFINE_CHAN 128\nBANDWIDTH_HZ 1280000\nSAMPLE_RATE 1280000\nMC_IP 0.0.0.0\nMC_PORT 0\nMC_SRC_IP 0.0.0.0\n" + for i, channel in enumerate(range(109, 132 + 1)): + metadata = input_file_metadata.format(channel, i + 1).encode('ascii') + metadata_padding = bytes(4096 - len(metadata)) + one_block = numpy.ones(256*64000*2,dtype=numpy.int8).tobytes() + zero_block = bytes(256 * 64000 * 2) + file_path = input_dir / f'1294797712_1294797712_{channel}.sub' + with open(file_path, 'wb') as file: + file.write(metadata) + file.write(metadata_padding) + file.write(zero_block) + for _ in range(160): + file.write(one_block) + + output_dir = working_dir / 'output_dir' + output_dir.mkdir(exist_ok=False, parents=True) + + result = run_application(run_script, input_dir, '1294797712', '1294797712', inverse_polyphase_filter_file, output_dir, 'true') + assert result.returncode == 0 + + + # Note: these tiles are flagged as faulty and will be skipped: 102, 115, 151, 164, 999, 2013, 2017, 2044, 2047 + tiles = [ + 51, 52, 53, 54, 55, 56, 57, 58, 71, 72, 73, 74, 75, 76, 77, 78, 101, 104, 105, + 106, 107, 108, 111, 112, 113, 114, 116, 117, 118, 121, 122, 123, 124, 125, 126, + 127, 128, 131, 132, 133, 134, 135, 136, 137, 138, 141, 142, 143, 144, 145, 146, 147, + 148, 152, 153, 154, 155, 156, 157, 158, 161, 162, 163, 165, 166, 167, 168, + 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, + 2015, 2016, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, + 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, + 2045, 2046, 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056 + ] + tile_output_filenames = [] + for tile in tiles: + tile_output_filenames.append(f'1294797712_1294797712_{tile}_X.bin') + tile_output_filenames.append(f'1294797712_1294797712_{tile}_Y.bin') + + log_file_name = '1294797712_1294797712_outputlog.txt' + + # Check there's output for all the tiles and polarisations, plus the log file. + output_dir_contents = [entry.name for entry in output_dir.iterdir()] + assert set(output_dir_contents) == set(tile_output_filenames + [log_file_name]) + + # Note: need to manually inspect output log file. + + convolved = numpy.convolve(numpy.ones(256,dtype=numpy.int8), filterVals, 'same') + timeDomain = numpy.fft.irfft(convolved) + + for filename in tile_output_filenames: + signal = read_output_signal(output_dir / filename) + # Check signal is all zeros. Note this checks the accuracy of the output signal to as high precision as is + # possible, stricter than the SRS requirement. + assert signal.min() == 0 and signal.max() == 0 + # Check that the output downsampling is as expected. The new sampling frequency is 54, manually calculated. + assert len(signal) == 160 * 64000 * 54 + del signal # Really don't want big array hanging around + From fcc6dd1fd699e9aab1eda344479a5d925076b356 Mon Sep 17 00:00:00 2001 From: Dylan Chanter Date: Fri, 15 Oct 2021 11:10:37 +0800 Subject: [PATCH 04/19] Updated fuctions added test for 4 signals with equasions from discord substring of expected data to compare --- test/integration/test_main.py | 56 ++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index b3e5abd..b99604c 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -9,17 +9,13 @@ # Assume we are in project root directory. TEST_DATA_PATH = Path('./test/input_data/integration') - -def test_54_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: - # Uses observation 1294797712, with signal data starting at 1294797712, - # signal with 54 as the input - - working_dir = working_dir / '54_signal_identity_ipfb' +def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: + # Uses observation 1294797712, with signal data starting at 1294797712 + working_dir = working_dir / '4_signals_identity_ipfb' working_dir.mkdir(exist_ok=False, parents=True) inv_polyphase_filter_path = working_dir / 'inverse_polyphase_filter.bin' - inv_polyphase_filter = numpy.zeros((13, 256), dtype=numpy.float32) - inv_polyphase_filter[:, -1] = 1 # Identity + inv_polyphase_filter = numpy.ones((1, 256), dtype=numpy.float32) write_inv_polyphase_filter(inv_polyphase_filter_path, inv_polyphase_filter) input_dir = working_dir / 'input_data' @@ -29,15 +25,17 @@ def test_54_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: "HDR_SIZE 4096\nPOPULATED 1\nOBS_ID 1294797712\nSUBOBS_ID 1294797712\nMODE VOLTAGE_START\n" \ "UTC_START 2021-01-16-02:01:34\nOBS_OFFSET 0\nNBIT 8\nNPOL 2\nNTIMESAMPLES 64000\nNINPUTS 256\n" \ "NINPUTS_XGPU 256\nAPPLY_PATH_WEIGHTS 0\nAPPLY_PATH_DELAYS 0\nINT_TIME_MSEC 500\nFSCRUNCH_FACTOR 50\n" \ - "APPLY_VIS_WEIGHTS 0\nTRANSFER_SIZE 5275648000\nPROJ_ID G0034\nEXPOSURE_SECS 304\nCOARSE_CHANNEL 113\n" \ - "CORR_COARSE_CHANNEL {}\nSECS_PER_SUBOBS 8\nUNIXTIME 1610762494\nUNIXTIME_MSEC 0\nFINE_CHAN_WIDTH_HZ 10000\n" \ + "APPLY_VIS_WEIGHTS 0\nTRANSFER_SIZE 5275648000\nPROJ_ID G0034\nEXPOSURE_SECS 304\nCOARSE_CHANNEL 116\n" \ + "CORR_COARSE_CHANNEL 116\nSECS_PER_SUBOBS 8\nUNIXTIME 1610762494\nUNIXTIME_MSEC 0\nFINE_CHAN_WIDTH_HZ 10000\n" \ "NFINE_CHAN 128\nBANDWIDTH_HZ 1280000\nSAMPLE_RATE 1280000\nMC_IP 0.0.0.0\nMC_PORT 0\nMC_SRC_IP 0.0.0.0\n" metadata = input_file_metadata_template.encode('ascii') metadata_padding = bytes(4096 - len(metadata)) - data_block = numpy.full((64000*256,2),(54,0),dtype=numpy.uint8) - print(data_block) - data_bytes_block = data_block.tobytes() + #equasions from discord + data_block1 = numpy.full((64000*256,2),(54,0),dtype=numpy.uint8).tobytes() + data_block2 = numpy.full((64000*256,2),(-80,0),dtype=numpy.uint8).tobytes() + data_block3 = numpy.full((64000*256,2),(173,8),dtype=numpy.uint8).tobytes() + data_block4 = numpy.full((64000*256,2),(2,-29),dtype=numpy.uint8).tobytes() zero_block = bytes(256 * 64000 * 2) file_path = input_dir / f'1294797712_1294797712_113.sub' with open(file_path, 'wb') as file: @@ -45,13 +43,33 @@ def test_54_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: file.write(metadata_padding) file.write(zero_block) for _ in range(160): - file.write(data_bytes_block) + file.write(data_block1) + file_path = input_dir / f'1294797712_1294797712_114.sub' + with open(file_path, 'wb') as file: + file.write(metadata) + file.write(metadata_padding) + file.write(zero_block) + for _ in range(160): + file.write(data_block2) + file_path = input_dir / f'1294797712_1294797712_116.sub' + with open(file_path, 'wb') as file: + file.write(metadata) + file.write(metadata_padding) + file.write(zero_block) + for _ in range(160): + file.write(data_block3) + file_path = input_dir / f'1294797712_1294797712_118.sub' + with open(file_path, 'wb') as file: + file.write(metadata) + file.write(metadata_padding) + file.write(zero_block) + for _ in range(160): + file.write(data_block4) output_dir = working_dir / 'output_dir' output_dir.mkdir(exist_ok=False, parents=True) result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'false') - assert result.returncode == 0 # Note: these tiles are flagged as faulty and will be skipped: 102, 115, 151, 164, 999, 2013, 2017, 2044, 2047 @@ -80,9 +98,8 @@ def test_54_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: for filename in tile_output_filenames: signal = read_output_signal(output_dir / filename) - assert signal.min() == 0 and signal.max() == 0 - # Check that the output downsampling is as expected. The new sampling frequency is 54, manually calculated. - assert len(signal) == 160 * 64000 * 54 + expected = [42,255,115,0,167,255,246,255,188,255,118,1,82,254,46,1,59,255,246,255,39,0,187,0] + assert set(expected) <= set(signal) del signal def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: @@ -163,3 +180,6 @@ def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: assert len(signal) == 160 * 64000 * 54 del signal # Really don't want big array hanging around + + + From 851d4ca6b49bf20e0a2a99e24887149c02c77616 Mon Sep 17 00:00:00 2001 From: Dylan Chanter Date: Fri, 15 Oct 2021 12:38:37 +0800 Subject: [PATCH 05/19] Updates to comparison at end of test --- test/integration/test_main.py | 91 ----------------------------------- 1 file changed, 91 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index b99604c..0711aba 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -9,98 +9,7 @@ # Assume we are in project root directory. TEST_DATA_PATH = Path('./test/input_data/integration') -def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: - # Uses observation 1294797712, with signal data starting at 1294797712 - working_dir = working_dir / '4_signals_identity_ipfb' - working_dir.mkdir(exist_ok=False, parents=True) - - inv_polyphase_filter_path = working_dir / 'inverse_polyphase_filter.bin' - inv_polyphase_filter = numpy.ones((1, 256), dtype=numpy.float32) - write_inv_polyphase_filter(inv_polyphase_filter_path, inv_polyphase_filter) - - input_dir = working_dir / 'input_data' - input_dir.mkdir(exist_ok=False, parents=True) - shutil.copyfile(TEST_DATA_PATH / '1294797712.metafits', input_dir / '1294797712.metafits') - input_file_metadata_template = \ - "HDR_SIZE 4096\nPOPULATED 1\nOBS_ID 1294797712\nSUBOBS_ID 1294797712\nMODE VOLTAGE_START\n" \ - "UTC_START 2021-01-16-02:01:34\nOBS_OFFSET 0\nNBIT 8\nNPOL 2\nNTIMESAMPLES 64000\nNINPUTS 256\n" \ - "NINPUTS_XGPU 256\nAPPLY_PATH_WEIGHTS 0\nAPPLY_PATH_DELAYS 0\nINT_TIME_MSEC 500\nFSCRUNCH_FACTOR 50\n" \ - "APPLY_VIS_WEIGHTS 0\nTRANSFER_SIZE 5275648000\nPROJ_ID G0034\nEXPOSURE_SECS 304\nCOARSE_CHANNEL 116\n" \ - "CORR_COARSE_CHANNEL 116\nSECS_PER_SUBOBS 8\nUNIXTIME 1610762494\nUNIXTIME_MSEC 0\nFINE_CHAN_WIDTH_HZ 10000\n" \ - "NFINE_CHAN 128\nBANDWIDTH_HZ 1280000\nSAMPLE_RATE 1280000\nMC_IP 0.0.0.0\nMC_PORT 0\nMC_SRC_IP 0.0.0.0\n" - - metadata = input_file_metadata_template.encode('ascii') - metadata_padding = bytes(4096 - len(metadata)) - #equasions from discord - data_block1 = numpy.full((64000*256,2),(54,0),dtype=numpy.uint8).tobytes() - data_block2 = numpy.full((64000*256,2),(-80,0),dtype=numpy.uint8).tobytes() - data_block3 = numpy.full((64000*256,2),(173,8),dtype=numpy.uint8).tobytes() - data_block4 = numpy.full((64000*256,2),(2,-29),dtype=numpy.uint8).tobytes() - zero_block = bytes(256 * 64000 * 2) - file_path = input_dir / f'1294797712_1294797712_113.sub' - with open(file_path, 'wb') as file: - file.write(metadata) - file.write(metadata_padding) - file.write(zero_block) - for _ in range(160): - file.write(data_block1) - file_path = input_dir / f'1294797712_1294797712_114.sub' - with open(file_path, 'wb') as file: - file.write(metadata) - file.write(metadata_padding) - file.write(zero_block) - for _ in range(160): - file.write(data_block2) - file_path = input_dir / f'1294797712_1294797712_116.sub' - with open(file_path, 'wb') as file: - file.write(metadata) - file.write(metadata_padding) - file.write(zero_block) - for _ in range(160): - file.write(data_block3) - file_path = input_dir / f'1294797712_1294797712_118.sub' - with open(file_path, 'wb') as file: - file.write(metadata) - file.write(metadata_padding) - file.write(zero_block) - for _ in range(160): - file.write(data_block4) - - output_dir = working_dir / 'output_dir' - output_dir.mkdir(exist_ok=False, parents=True) - - result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'false') - assert result.returncode == 0 - - # Note: these tiles are flagged as faulty and will be skipped: 102, 115, 151, 164, 999, 2013, 2017, 2044, 2047 - tiles = [ - 51, 52, 53, 54, 55, 56, 57, 58, 71, 72, 73, 74, 75, 76, 77, 78, 101, 104, 105, - 106, 107, 108, 111, 112, 113, 114, 116, 117, 118, 121, 122, 123, 124, 125, 126, - 127, 128, 131, 132, 133, 134, 135, 136, 137, 138, 141, 142, 143, 144, 145, 146, 147, - 148, 152, 153, 154, 155, 156, 157, 158, 161, 162, 163, 165, 166, 167, 168, - 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, - 2015, 2016, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, - 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, - 2045, 2046, 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056 - ] - tile_output_filenames = [] - for tile in tiles: - tile_output_filenames.append(f'1294797712_1294797712_{tile}_X.bin') - tile_output_filenames.append(f'1294797712_1294797712_{tile}_Y.bin') - - log_file_name = '1294797712_1294797712_outputlog.txt' - # Check there's output for all the tiles and polarisations, plus the log file. - output_dir_contents = [entry.name for entry in output_dir.iterdir()] - assert set(output_dir_contents) == set(tile_output_filenames + [log_file_name]) - - # Note: need to manually inspect output log file. - - for filename in tile_output_filenames: - signal = read_output_signal(output_dir / filename) - expected = [42,255,115,0,167,255,246,255,188,255,118,1,82,254,46,1,59,255,246,255,39,0,187,0] - assert set(expected) <= set(signal) - del signal def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: # Test of valid input data, with all zeros signal data and the identity inverse polyphase filter. From 4ec2bb37c93bd1bec8da2d298a0ad3b94655b875 Mon Sep 17 00:00:00 2001 From: Dylan Chanter Date: Fri, 15 Oct 2021 12:40:17 +0800 Subject: [PATCH 06/19] Merged in dev branch + updated comparision --- test/integration/test_main.py | 93 +++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index 36916ab..23a0227 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -11,7 +11,100 @@ # Assume we are in project root directory. TEST_DATA_PATH = Path('./test/input_data/integration') +def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: + # Uses observation 1294797712, with signal data starting at 1294797712 + working_dir = working_dir / '4_signals_identity_ipfb' + working_dir.mkdir(exist_ok=False, parents=True) + + inv_polyphase_filter_path = working_dir / 'inverse_polyphase_filter.bin' + inv_polyphase_filter = numpy.ones((1, 256), dtype=numpy.float32) + write_inv_polyphase_filter(inv_polyphase_filter_path, inv_polyphase_filter) + + input_dir = working_dir / 'input_data' + input_dir.mkdir(exist_ok=False, parents=True) + shutil.copyfile(TEST_DATA_PATH / '1294797712.metafits', input_dir / '1294797712.metafits') + input_file_metadata_template = \ + "HDR_SIZE 4096\nPOPULATED 1\nOBS_ID 1294797712\nSUBOBS_ID 1294797712\nMODE VOLTAGE_START\n" \ + "UTC_START 2021-01-16-02:01:34\nOBS_OFFSET 0\nNBIT 8\nNPOL 2\nNTIMESAMPLES 64000\nNINPUTS 256\n" \ + "NINPUTS_XGPU 256\nAPPLY_PATH_WEIGHTS 0\nAPPLY_PATH_DELAYS 0\nINT_TIME_MSEC 500\nFSCRUNCH_FACTOR 50\n" \ + "APPLY_VIS_WEIGHTS 0\nTRANSFER_SIZE 5275648000\nPROJ_ID G0034\nEXPOSURE_SECS 304\nCOARSE_CHANNEL 116\n" \ + "CORR_COARSE_CHANNEL 116\nSECS_PER_SUBOBS 8\nUNIXTIME 1610762494\nUNIXTIME_MSEC 0\nFINE_CHAN_WIDTH_HZ 10000\n" \ + "NFINE_CHAN 128\nBANDWIDTH_HZ 1280000\nSAMPLE_RATE 1280000\nMC_IP 0.0.0.0\nMC_PORT 0\nMC_SRC_IP 0.0.0.0\n" + + metadata = input_file_metadata_template.encode('ascii') + metadata_padding = bytes(4096 - len(metadata)) + #equasions from discord + data_block1 = numpy.full((64000*256,2),(54,0),dtype=numpy.uint8).tobytes() + data_block2 = numpy.full((64000*256,2),(-80,0),dtype=numpy.uint8).tobytes() + data_block3 = numpy.full((64000*256,2),(173,8),dtype=numpy.uint8).tobytes() + data_block4 = numpy.full((64000*256,2),(2,-29),dtype=numpy.uint8).tobytes() + zero_block = bytes(256 * 64000 * 2) + file_path = input_dir / f'1294797712_1294797712_113.sub' + with open(file_path, 'wb') as file: + file.write(metadata) + file.write(metadata_padding) + file.write(zero_block) + for _ in range(160): + file.write(data_block1) + file_path = input_dir / f'1294797712_1294797712_114.sub' + with open(file_path, 'wb') as file: + file.write(metadata) + file.write(metadata_padding) + file.write(zero_block) + for _ in range(160): + file.write(data_block2) + file_path = input_dir / f'1294797712_1294797712_116.sub' + with open(file_path, 'wb') as file: + file.write(metadata) + file.write(metadata_padding) + file.write(zero_block) + for _ in range(160): + file.write(data_block3) + file_path = input_dir / f'1294797712_1294797712_118.sub' + with open(file_path, 'wb') as file: + file.write(metadata) + file.write(metadata_padding) + file.write(zero_block) + for _ in range(160): + file.write(data_block4) + + output_dir = working_dir / 'output_dir' + output_dir.mkdir(exist_ok=False, parents=True) + + result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'false') + assert result.returncode == 0 + # Note: these tiles are flagged as faulty and will be skipped: 102, 115, 151, 164, 999, 2013, 2017, 2044, 2047 + tiles = [ + 51, 52, 53, 54, 55, 56, 57, 58, 71, 72, 73, 74, 75, 76, 77, 78, 101, 104, 105, + 106, 107, 108, 111, 112, 113, 114, 116, 117, 118, 121, 122, 123, 124, 125, 126, + 127, 128, 131, 132, 133, 134, 135, 136, 137, 138, 141, 142, 143, 144, 145, 146, 147, + 148, 152, 153, 154, 155, 156, 157, 158, 161, 162, 163, 165, 166, 167, 168, + 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, + 2015, 2016, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, + 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, + 2045, 2046, 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056 + ] + tile_output_filenames = [] + for tile in tiles: + tile_output_filenames.append(f'1294797712_1294797712_{tile}_X.bin') + tile_output_filenames.append(f'1294797712_1294797712_{tile}_Y.bin') + + log_file_name = '1294797712_1294797712_outputlog.txt' + + # Check there's output for all the tiles and polarisations, plus the log file. + output_dir_contents = [entry.name for entry in output_dir.iterdir()] + assert set(output_dir_contents) == set(tile_output_filenames + [log_file_name]) + + # Note: need to manually inspect output log file. + + for filename in tile_output_filenames: + signal = read_output_signal(output_dir / filename) + expected = [42,255,115,0,167,255,246,255,188,255,118,1,82,254,46,1,59,255,246,255,39,0,187,0]*64000*160 + signallist = signal.tolist() + assert signallist == expected + del signal + def test_invalid_command_line_arguments(run_script: Path, working_dir: Path) -> None: working_dir = working_dir / 'invalid_command_line_arguments' working_dir.mkdir(exist_ok=False, parents=True) From 263d249251698ae5f5bc45ddb9270e0f0153d365 Mon Sep 17 00:00:00 2001 From: Dylan Chanter Date: Fri, 15 Oct 2021 15:08:44 +0800 Subject: [PATCH 07/19] Fixed meta data errors for correct channle meta data was cuasing errors --- test/integration/test_main.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index 23a0227..f0a6623 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -27,8 +27,8 @@ def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: "HDR_SIZE 4096\nPOPULATED 1\nOBS_ID 1294797712\nSUBOBS_ID 1294797712\nMODE VOLTAGE_START\n" \ "UTC_START 2021-01-16-02:01:34\nOBS_OFFSET 0\nNBIT 8\nNPOL 2\nNTIMESAMPLES 64000\nNINPUTS 256\n" \ "NINPUTS_XGPU 256\nAPPLY_PATH_WEIGHTS 0\nAPPLY_PATH_DELAYS 0\nINT_TIME_MSEC 500\nFSCRUNCH_FACTOR 50\n" \ - "APPLY_VIS_WEIGHTS 0\nTRANSFER_SIZE 5275648000\nPROJ_ID G0034\nEXPOSURE_SECS 304\nCOARSE_CHANNEL 116\n" \ - "CORR_COARSE_CHANNEL 116\nSECS_PER_SUBOBS 8\nUNIXTIME 1610762494\nUNIXTIME_MSEC 0\nFINE_CHAN_WIDTH_HZ 10000\n" \ + "APPLY_VIS_WEIGHTS 0\nTRANSFER_SIZE 5275648000\nPROJ_ID G0034\nEXPOSURE_SECS 304\nCOARSE_CHANNEL {}\n" \ + "CORR_COARSE_CHANNEL {}\nSECS_PER_SUBOBS 8\nUNIXTIME 1610762494\nUNIXTIME_MSEC 0\nFINE_CHAN_WIDTH_HZ 10000\n" \ "NFINE_CHAN 128\nBANDWIDTH_HZ 1280000\nSAMPLE_RATE 1280000\nMC_IP 0.0.0.0\nMC_PORT 0\nMC_SRC_IP 0.0.0.0\n" metadata = input_file_metadata_template.encode('ascii') @@ -39,6 +39,7 @@ def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: data_block3 = numpy.full((64000*256,2),(173,8),dtype=numpy.uint8).tobytes() data_block4 = numpy.full((64000*256,2),(2,-29),dtype=numpy.uint8).tobytes() zero_block = bytes(256 * 64000 * 2) + metadata = input_file_metadata_template.format(113,1).encode('ascii') file_path = input_dir / f'1294797712_1294797712_113.sub' with open(file_path, 'wb') as file: file.write(metadata) @@ -46,20 +47,23 @@ def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: file.write(zero_block) for _ in range(160): file.write(data_block1) + metadata = input_file_metadata_template.format(114,2).encode('ascii') file_path = input_dir / f'1294797712_1294797712_114.sub' with open(file_path, 'wb') as file: file.write(metadata) file.write(metadata_padding) file.write(zero_block) for _ in range(160): - file.write(data_block2) + file.write(data_block2) + metadata = input_file_metadata_template.format(116,3).encode('ascii') file_path = input_dir / f'1294797712_1294797712_116.sub' with open(file_path, 'wb') as file: file.write(metadata) file.write(metadata_padding) file.write(zero_block) for _ in range(160): - file.write(data_block3) + file.write(data_block3) + metadata = input_file_metadata_template.format(118,4).encode('ascii') file_path = input_dir / f'1294797712_1294797712_118.sub' with open(file_path, 'wb') as file: file.write(metadata) @@ -104,7 +108,7 @@ def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: signallist = signal.tolist() assert signallist == expected del signal - + def test_invalid_command_line_arguments(run_script: Path, working_dir: Path) -> None: working_dir = working_dir / 'invalid_command_line_arguments' working_dir.mkdir(exist_ok=False, parents=True) From 285afb287986414373f9ac9bb353bc71bb494d05 Mon Sep 17 00:00:00 2001 From: Dylan Chanter Date: Fri, 15 Oct 2021 22:10:29 +0800 Subject: [PATCH 08/19] Works :) --- test/integration/test_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index f0a6623..7918b02 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -75,7 +75,7 @@ def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: output_dir = working_dir / 'output_dir' output_dir.mkdir(exist_ok=False, parents=True) - result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'false') + result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'true') assert result.returncode == 0 # Note: these tiles are flagged as faulty and will be skipped: 102, 115, 151, 164, 999, 2013, 2017, 2044, 2047 @@ -104,7 +104,7 @@ def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: for filename in tile_output_filenames: signal = read_output_signal(output_dir / filename) - expected = [42,255,115,0,167,255,246,255,188,255,118,1,82,254,46,1,59,255,246,255,39,0,187,0]*64000*160 + expected = [42,255,115,0,167,255,246,255,188,255,118,1,82,254,46,1,59,255,246,255,39,0,187,0]*10240000 signallist = signal.tolist() assert signallist == expected del signal From 2d3ba7ff56e5424e8833882772cb426ae88ddb76 Mon Sep 17 00:00:00 2001 From: Zak Booth Date: Mon, 18 Oct 2021 14:48:48 +0800 Subject: [PATCH 09/19] CP6-82 Added raw sampling frequency (integer length of Fourier transform) to output log file writer Also fixed typo in main. --- src/Main.cpp | 2 +- src/OutputLogFileWriter.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Main.cpp b/src/Main.cpp index fb9e585..8732255 100644 --- a/src/Main.cpp +++ b/src/Main.cpp @@ -180,7 +180,7 @@ void runNode(PrimaryNodeCommunicator& primary, int argc, char* argv[]) { std::cerr << "Node 0 (Primary): " << e.what() << std::endl; } - std::cout << "Node 0 (Primary): Terminated succesfully" << std::endl; + std::cout << "Node 0 (Primary): Terminated successfully" << std::endl; } diff --git a/src/OutputLogFileWriter.cpp b/src/OutputLogFileWriter.cpp index 6b7c8f4..7a45422 100644 --- a/src/OutputLogFileWriter.cpp +++ b/src/OutputLogFileWriter.cpp @@ -59,6 +59,7 @@ void writeProcessingDetails(std::ofstream& log, ChannelRemapping const& channelR double samplingPeriod = (1.0 / sampleRate) * 1000.0; log << "SIGNAL PROCESSING DETAILS" << std::endl; + log << "Output raw sampling frequency: " << channelRemapping.newSamplingFreq << std::endl; log << "Output sample rate: " << roundThreeDecimalPlace(sampleRate) << " MHz" << std::endl; log << "Output sampling period: " << roundThreeDecimalPlace(samplingPeriod) << " ns\n" << std::endl; } From 4a84a8035a20f24b7b063fba5f83d543d2d664ab Mon Sep 17 00:00:00 2001 From: Zak Booth Date: Mon, 18 Oct 2021 17:29:52 +0800 Subject: [PATCH 10/19] CP6-82 Changed wording for new output frequency --- src/OutputLogFileWriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputLogFileWriter.cpp b/src/OutputLogFileWriter.cpp index 7a45422..2e4fb68 100644 --- a/src/OutputLogFileWriter.cpp +++ b/src/OutputLogFileWriter.cpp @@ -59,7 +59,7 @@ void writeProcessingDetails(std::ofstream& log, ChannelRemapping const& channelR double samplingPeriod = (1.0 / sampleRate) * 1000.0; log << "SIGNAL PROCESSING DETAILS" << std::endl; - log << "Output raw sampling frequency: " << channelRemapping.newSamplingFreq << std::endl; + log << "Integer length of Fourier transform: " << channelRemapping.newSamplingFreq << std::endl; log << "Output sample rate: " << roundThreeDecimalPlace(sampleRate) << " MHz" << std::endl; log << "Output sampling period: " << roundThreeDecimalPlace(samplingPeriod) << " ns\n" << std::endl; } From e53a8fcd96779dc189742b5fd57769002d0bb987 Mon Sep 17 00:00:00 2001 From: Reece Jones Date: Wed, 20 Oct 2021 11:58:42 +0800 Subject: [PATCH 11/19] Improvements to readme based on client feedback. --- README.md | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 64eedee..c7cb9a2 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ Please see the sections below for detailed instruction on how to build and run t For building: - Docker. Any recent version with support for multi-stage builds should work, e.g. Docker Engine 20.x. -- Singularity. Targeted version 3.5.3. (Not required for building on personal machine.) +- Singularity. Targeted version 3.5.3. (Not required if building ONLY for use with Docker on personal machine.) Please also see the "Installing Singularity" section. +- At least 20GB free disk space. For running on Garrawarla: @@ -53,6 +54,9 @@ For integration testing: - Pytest. Targeted version 4.6.9. - `mwatdr_utils`. Please see `mwatdr_utils/README.md` for its requirements. +Note: "targeted version" means the version the software is guaranteed to work with. +Similar versions that aren't significantly older or newer may also work. + ## Building and Running `main` for Use on Garrawarla The instructions here are for building and running the main application for use on Garrawarla - if you are a standard user, this is probably what you want to do. @@ -72,7 +76,10 @@ By default, Docker build will use base images which match the current system arc You can modify the Dockerfile to manually specify the architecture if you wish to build on other types of systems. The build will probably take quite a while the first time, due to some large libraries which are installed and built in the image. -Do not be alarmed if it takes over 20 minutes. +Do not be alarmed if it takes over 20 minutes. +The resulting image is approximately 5.5GB in size. + +Please note that at this stage, the built image is stored internally within Docker - no image file is produced (that is the next step). Next, you need to convert the Docker image to a Singularity image. There are a few options for which system to perform the conversion on: @@ -80,6 +87,8 @@ There are a few options for which system to perform the conversion on: - Use Singularity installed on your personal machine. Installation of Singularity is easy if you use Linux or Mac, but difficult for Windows. - Use Singularity already installed on Garrawarla, either on the login nodes or on the compute nodes with an interactive SLURM job. Using the compute nodes is likely fastest to build, but it does require transferring the >5GB Docker image to Garrawarla first. +Please see the "Installing Singularity" section for further details on installing Singularity. + Convert the Docker image to tar format: ```bash @@ -97,7 +106,8 @@ This takes the `main.tar` Docker image you just saved and produces `main.sif`, t By default, Singularity uses the system temporary directory for working space, and may use a few gigabytes of space, which may be an issue for some systems - particularly Linux, where `/tmp` may be in RAM. Therefore it is highly recommended to set `` to somewhere on disk. -The Singularity build may take several minutes to complete. +The Singularity build may take several minutes to complete. +The resulting image will be approximately 1.5GB in size. ### Running @@ -138,6 +148,25 @@ Then simply run the `main` target with Docker, using the provided script: `` are the application command line arguments. Please see the "User Interface" section for details. +## Installing Singularity + +If you are building the application to run on Garrawarla, you will need to have Singularity installed on whichever machine you use to perform the build. +The difficulty of installation of Singularity depends on your operating system. + +### Linux + +Instructions can be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-linux) from Singularity and [here](https://pawseysc.github.io/singularity-containers/44-setup-singularity/index.html) from Pawsey. + +### Mac + +For installing Singularity on Mac, we recommend using [Brew](https://brew.sh/). +Installation is then as simple as installing the [Singularity Brew package](https://formulae.brew.sh/formula/singularity#default) (a single terminal command). + +### Windows + +Installation on Windows is the most difficult, and thus we recommend not using Windows to build the application with Singularity if possible. +Instructions may be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-windows-or-mac) from Singularity. + ## User Interface The `main` target has the following positional command line arguments: From 7d2a651918c50a61fcfaf3006c761dd5dbd5c7fb Mon Sep 17 00:00:00 2001 From: Reece Jones Date: Wed, 20 Oct 2021 14:44:32 +0800 Subject: [PATCH 12/19] Further improving readability of readme. --- README.md | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c7cb9a2..75e87b5 100644 --- a/README.md +++ b/README.md @@ -63,33 +63,29 @@ The instructions here are for building and running the main application for use ### Building -First, build the `main` target with Docker on your personal machine (since Garrawarla does not have Docker), using the provided script: +First, we will build the `main` target with Docker on your personal machine, since Garrawarla does not have Docker. + +Note that you must perform the following step on an x86 machine, to match the CPU architecture of Garrawarla. +(By default, Docker build will use base images which match the current system architecture. +If you wish to build on other types of systems, you can modify the Dockerfile to explicitly specify the architecture.) + +Run the provided build script as follows: ```bash ./docker_build.sh main Release garrawarla singularity ``` -This invokes Docker to build the main application image, using a configuration optimised for high performance on Garrawarla, and designed to run with Singularity. -The name of the image is `mwatdr/main`. -Note that you must perform this step on an x86 machine, to match the CPU architecture of Garrawarla. -By default, Docker build will use base images which match the current system architecture. -You can modify the Dockerfile to manually specify the architecture if you wish to build on other types of systems. - -The build will probably take quite a while the first time, due to some large libraries which are installed and built in the image. -Do not be alarmed if it takes over 20 minutes. -The resulting image is approximately 5.5GB in size. - -Please note that at this stage, the built image is stored internally within Docker - no image file is produced (that is the next step). +This invokes Docker to build the main application image, using a configuration optimised for high performance on Garrawarla, and designed to run with Singularity. It will probably take quite a while the first time, due to some large libraries which are installed and built in the image. +The resulting image is named `mwatdr/main` and is approximately 5.5GB in size. +At this stage, the image is stored internally within Docker - no visible file output is produced. Next, you need to convert the Docker image to a Singularity image. There are a few options for which system to perform the conversion on: -- Use Singularity installed on your personal machine. Installation of Singularity is easy if you use Linux or Mac, but difficult for Windows. +- Use Singularity installed on your personal machine. Installation of Singularity is fairly easy if you use Linux or Mac, but difficult for Windows. Please see the "Installing Singularity" section for details. - Use Singularity already installed on Garrawarla, either on the login nodes or on the compute nodes with an interactive SLURM job. Using the compute nodes is likely fastest to build, but it does require transferring the >5GB Docker image to Garrawarla first. -Please see the "Installing Singularity" section for further details on installing Singularity. - -Convert the Docker image to tar format: +Convert the Docker image to a tar file: ```bash docker save mwatdr/main -o main.tar @@ -125,13 +121,13 @@ sbatch slurm_main.sh ## Building and Running `main` for Use on Personal Machine +The instructions here are for building and running the main application for use on your personal, standard computer. +This may be useful for testing purposes. + NOTE: The application is designed to run on Garrawarla, and your personal machine may not be powerful enough to feasibly run the `main` target. For 24 frequency channels and 256 antenna inputs, we found `main` requires approximately 8GB of memory per process, and running with a few processes will take on the order of a few hours to complete. It is possible to manually specify the number of processes to `mpirun` in `entrypoint.sh`, but there will be a tradeoff between memory usage and run time. -The instructions here are for building and running the main application for use on your personal, standard computer. -This may be useful for testing purposes. - First, build the `main` target with Docker, using the provided script: ```bash @@ -155,17 +151,17 @@ The difficulty of installation of Singularity depends on your operating system. ### Linux -Instructions can be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-linux) from Singularity and [here](https://pawseysc.github.io/singularity-containers/44-setup-singularity/index.html) from Pawsey. +Instructions can be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-linux) from Singularity and [here](https://pawseysc.github.io/singularity-containers/44-setup-singularity/index.html) from Pawsey. Depending on your distro, it may be as simple as installing the Singularity package from your package manager. ### Mac For installing Singularity on Mac, we recommend using [Brew](https://brew.sh/). -Installation is then as simple as installing the [Singularity Brew package](https://formulae.brew.sh/formula/singularity#default) (a single terminal command). +Installation is then as simple as installing the [Singularity Brew package](https://formulae.brew.sh/formula/singularity#default). ### Windows Installation on Windows is the most difficult, and thus we recommend not using Windows to build the application with Singularity if possible. -Instructions may be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-windows-or-mac) from Singularity. +Instructions from Singularity may be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-windows-or-mac). ## User Interface From 9ace925522dfb3303b1ce71fe51cc182e12a1c54 Mon Sep 17 00:00:00 2001 From: Reece Jones Date: Wed, 20 Oct 2021 16:17:56 +0800 Subject: [PATCH 13/19] Fixed Singularity install instructions in readme. --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 75e87b5..f5310a8 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,8 @@ At this stage, the image is stored internally within Docker - no visible file ou Next, you need to convert the Docker image to a Singularity image. There are a few options for which system to perform the conversion on: -- Use Singularity installed on your personal machine. Installation of Singularity is fairly easy if you use Linux or Mac, but difficult for Windows. Please see the "Installing Singularity" section for details. -- Use Singularity already installed on Garrawarla, either on the login nodes or on the compute nodes with an interactive SLURM job. Using the compute nodes is likely fastest to build, but it does require transferring the >5GB Docker image to Garrawarla first. +- Use Singularity installed on your personal machine. The difficulty of installation depends on your operating system. Please see the "Installing Singularity" section for details. +- Use Singularity already installed on Garrawarla, either on the login nodes or on the compute nodes with an interactive SLURM job. However, it does require transferring the >5GB Docker image to Garrawarla first. This is our recommended method if you have a good internet connection. Convert the Docker image to a tar file: @@ -147,21 +147,23 @@ Then simply run the `main` target with Docker, using the provided script: ## Installing Singularity If you are building the application to run on Garrawarla, you will need to have Singularity installed on whichever machine you use to perform the build. -The difficulty of installation of Singularity depends on your operating system. +If you choose not to build on Garrawarla, you will need to install Singularity yourself. ### Linux -Instructions can be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-linux) from Singularity and [here](https://pawseysc.github.io/singularity-containers/44-setup-singularity/index.html) from Pawsey. Depending on your distro, it may be as simple as installing the Singularity package from your package manager. +Depending on your distro, prebuilt Singularity packages may be available for simple installation from your package manager. Try searching for "singularity" or "singularity-container". + +Alternatively, you may need to build from source. Instructions can be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-linux) from Singularity and [here](https://pawseysc.github.io/singularity-containers/44-setup-singularity/index.html) from Pawsey. ### Mac -For installing Singularity on Mac, we recommend using [Brew](https://brew.sh/). -Installation is then as simple as installing the [Singularity Brew package](https://formulae.brew.sh/formula/singularity#default). +Installation on Mac may be done by building Singularity from source, or using a Vagrant virtual machine. +Instructions can be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-windows-or-mac). ### Windows -Installation on Windows is the most difficult, and thus we recommend not using Windows to build the application with Singularity if possible. -Instructions from Singularity may be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-windows-or-mac). +Installation on Windows may be done by building Singularity from source, or using a Vagrant virtual machine. +Instructions can be found [here](https://sylabs.io/guides/3.8/admin-guide/installation.html#installation-on-windows-or-mac). ## User Interface From 944c60ffce779c6bcd97fe8526cf36ff9cea1d44 Mon Sep 17 00:00:00 2001 From: Benjamin Bowers Date: Wed, 20 Oct 2021 16:48:38 +0800 Subject: [PATCH 14/19] Made changes based on comments --- test/integration/test_main.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index ea8d8bc..1d1f24f 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -478,7 +478,7 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: metadata = input_file_metadata_template.format(channel, i + 1).encode('ascii') metadata_padding = bytes(4096 - len(metadata)) zero_block = bytes(256 * 64000 * 2) - one_block = numpy.full((64000*256,2), (1, 0), dtype=numpy.uint8).tobytes() + one_block = numpy.full((64000*256,2), (1, 2), dtype=numpy.uint8).tobytes() file_path = input_dir / f'1294797712_1294797712_{channel}.sub' with open(file_path, 'wb') as file: file.write(metadata) @@ -520,7 +520,7 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: # Note: need to manually inspect output log file. remapped_row = numpy.zeros(28, dtype=numpy.complex64) - remapped_row[12] = 1.0+0j + remapped_row[12] = 1.0+2.0j remapped_arr = numpy.full((160*6400,28), remapped_row, dtype=numpy.complex64) remapped_arr[:,12] = numpy.convolve(remapped_arr[:,12], filterVals, 'same') timeDomain = numpy.zeros((6400*160,28), dtype=numpy.int16) @@ -531,8 +531,10 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: signal = read_output_signal(output_dir / filename) # Check that the output downsampling is as expected. The new sampling frequency is 54, manually calculated. assert len(signal) == 160 * 64000 * 54 - remapped_row = numpy.zeros(28, dtype=numpy.complex64) - assert len(signal) == 160 * 64000 * 54 + + error_sum = 0 for ii in range(160*6400*54): - assert pow(abs(float(signal[ii]) - float(timeDomain[ii])), 2)/pow(timeDomain[ii], 2) <= 0.005 + error_sum = error_sum + pow(abs(signal[ii] - timeDomain[ii]), 2)/pow(abs(timeDomain[ii]), 2) + assert error_sum/(160*6400*54) <= 0.005 + del signal # Really don't want big array hanging around From 9fbf5b0c4f6fa0c6105e7ff970d2e2f5fd2a8d0b Mon Sep 17 00:00:00 2001 From: Benjamin Bowers Date: Wed, 20 Oct 2021 17:56:52 +0800 Subject: [PATCH 15/19] Added second accuracy check --- test/integration/test_main.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index 1d1f24f..92458bb 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -349,8 +349,7 @@ def test_all_one_pfb(run_script: Path, working_dir: Path) -> None: 148, 152, 153, 154, 155, 156, 157, 158, 161, 162, 163, 165, 166, 167, 168, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2015, 2016, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, - 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, - 2045, 2046, 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056 + 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, 2043, 2045, 2046, 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056 ] tile_output_filenames = [] for tile in tiles: @@ -449,7 +448,7 @@ def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: del signal # Really don't want big array hanging around def test_complex_pfb(run_script: Path, working_dir: Path) -> None: - # Test of valid input data, with all ones signal data and the polyphase filter is to be all zeros. + # Test of all one input data with a pfb that contains complex values on channel 120 working_dir = working_dir / 'one_signal_complex_pfb' working_dir.mkdir(exist_ok=False, parents=True) @@ -463,7 +462,7 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: write_inv_polyphase_filter(inv_polyphase_filter_path, inv_polyphase_filter) input_dir = working_dir / 'input_data' - input_dir.mkdir(exist_ok=False, parents=True) + input_dir.mkdir(mode=777, exist_ok=False, parents=True) shutil.copyfile(TEST_DATA_PATH / '1294797712.metafits', input_dir / '1294797712.metafits') input_file_metadata_template = \ "HDR_SIZE 4096\nPOPULATED 1\nOBS_ID 1294797712\nSUBOBS_ID 1294797712\nMODE VOLTAGE_START\n" \ @@ -490,7 +489,7 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: output_dir = working_dir / 'output_dir' - output_dir.mkdir(exist_ok=False, parents=True) + output_dir.mkdir(mode=777, exist_ok=False, parents=True) result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'true') assert result.returncode == 0 @@ -521,20 +520,27 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: # Note: need to manually inspect output log file. remapped_row = numpy.zeros(28, dtype=numpy.complex64) remapped_row[12] = 1.0+2.0j - remapped_arr = numpy.full((160*6400,28), remapped_row, dtype=numpy.complex64) + remapped_arr = numpy.full((160*64000,28), remapped_row, dtype=numpy.complex64) remapped_arr[:,12] = numpy.convolve(remapped_arr[:,12], filterVals, 'same') - timeDomain = numpy.zeros((6400*160,28), dtype=numpy.int16) - for ii in range(160*6400) : + timeDomain = numpy.zeros((64000*160,54), dtype=numpy.int16) + for ii in range(160*64000) : timeDomain[ii] = numpy.fft.irfft(remapped_arr[ii], norm="forward") + timeDomain = timeDomain.flatten() + std_deviation = numpy.std(timeDomain) + for filename in tile_output_filenames: signal = read_output_signal(output_dir / filename) # Check that the output downsampling is as expected. The new sampling frequency is 54, manually calculated. assert len(signal) == 160 * 64000 * 54 - error_sum = 0 - for ii in range(160*6400*54): - error_sum = error_sum + pow(abs(signal[ii] - timeDomain[ii]), 2)/pow(abs(timeDomain[ii]), 2) - assert error_sum/(160*6400*54) <= 0.005 + deviation_count = 0 + for ii in range(160*64000*54): + abs_diff = abs(signal[ii]-timeDomain[ii]) + error_sum = error_sum + pow(abs_diff, 2)/pow(abs(timeDomain[ii]), 2) + if ( abs_diff > std_deviation ): + deviation_count = deviation_count + 1 + assert deviation_count/(160*6400*54) <= 0.00001 + assert error_sum/(160*64000*54) <= 0.005 del signal # Really don't want big array hanging around From ddfe968f9e5b029eda8d1f8b8ac7ea6878f4d2f9 Mon Sep 17 00:00:00 2001 From: Benjamin Bowers Date: Wed, 20 Oct 2021 18:24:14 +0800 Subject: [PATCH 16/19] Improved numpy comparisons --- test/integration/test_main.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index 92458bb..f29d72c 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -527,20 +527,15 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: timeDomain[ii] = numpy.fft.irfft(remapped_arr[ii], norm="forward") timeDomain = timeDomain.flatten() - std_deviation = numpy.std(timeDomain) for filename in tile_output_filenames: signal = read_output_signal(output_dir / filename) # Check that the output downsampling is as expected. The new sampling frequency is 54, manually calculated. assert len(signal) == 160 * 64000 * 54 - error_sum = 0 - deviation_count = 0 - for ii in range(160*64000*54): - abs_diff = abs(signal[ii]-timeDomain[ii]) - error_sum = error_sum + pow(abs_diff, 2)/pow(abs(timeDomain[ii]), 2) - if ( abs_diff > std_deviation ): - deviation_count = deviation_count + 1 - assert deviation_count/(160*6400*54) <= 0.00001 - assert error_sum/(160*64000*54) <= 0.005 + errors = signal - timeDomain + norm_sq_errors = (errors * errors) / (timeDomain * timeDomain) + assert numpy.mean(norm_sq_errors) <= 0.005 + deviation_prob = numpy.mean(numpy.abs(errors) > numpy.std(timeDomain)) + assert deviation_prob <= 1e-6 del signal # Really don't want big array hanging around From 3aaccf55eecbd078224b560cbd4f85a8c627248a Mon Sep 17 00:00:00 2001 From: Benjamin Bowers Date: Wed, 20 Oct 2021 20:28:44 +0800 Subject: [PATCH 17/19] Minor changes to get working on garrawarla and it's ancient numpy version --- test/integration/test_main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index f29d72c..5ded6a2 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -324,7 +324,7 @@ def test_all_one_pfb(run_script: Path, working_dir: Path) -> None: for i, channel in enumerate(range(109, 132 + 1)): metadata = input_file_metadata.format(channel, i + 1).encode('ascii') metadata_padding = bytes(4096 - len(metadata)) - one_block = numpy.ones(256*64000*2,dtype=numpy.int8).to_bytes() + one_block = numpy.ones(256*64000*2,dtype=numpy.int8).tobytes() zero_block = bytes(256 * 64000 * 2) file_path = input_dir / f'1294797712_1294797712_{channel}.sub' with open(file_path, 'wb') as file: @@ -462,7 +462,7 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: write_inv_polyphase_filter(inv_polyphase_filter_path, inv_polyphase_filter) input_dir = working_dir / 'input_data' - input_dir.mkdir(mode=777, exist_ok=False, parents=True) + input_dir.mkdir(exist_ok=False, parents=True) shutil.copyfile(TEST_DATA_PATH / '1294797712.metafits', input_dir / '1294797712.metafits') input_file_metadata_template = \ "HDR_SIZE 4096\nPOPULATED 1\nOBS_ID 1294797712\nSUBOBS_ID 1294797712\nMODE VOLTAGE_START\n" \ @@ -489,7 +489,7 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: output_dir = working_dir / 'output_dir' - output_dir.mkdir(mode=777, exist_ok=False, parents=True) + output_dir.mkdir(exist_ok=False, parents=True) result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'true') assert result.returncode == 0 @@ -524,7 +524,7 @@ def test_complex_pfb(run_script: Path, working_dir: Path) -> None: remapped_arr[:,12] = numpy.convolve(remapped_arr[:,12], filterVals, 'same') timeDomain = numpy.zeros((64000*160,54), dtype=numpy.int16) for ii in range(160*64000) : - timeDomain[ii] = numpy.fft.irfft(remapped_arr[ii], norm="forward") + timeDomain[ii] = numpy.fft.irfft(remapped_arr[ii], norm=None)*54 timeDomain = timeDomain.flatten() From 6cac80fe279ea891f8965d6535c0230b9c4ef7e7 Mon Sep 17 00:00:00 2001 From: Benjamin Bowers Date: Wed, 20 Oct 2021 20:59:34 +0800 Subject: [PATCH 18/19] Working Test case in theory --- test/integration/test_main.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index 7918b02..d49a5e7 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -47,30 +47,30 @@ def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: file.write(zero_block) for _ in range(160): file.write(data_block1) - metadata = input_file_metadata_template.format(114,2).encode('ascii') - file_path = input_dir / f'1294797712_1294797712_114.sub' + metadata = input_file_metadata_template.format(114,2).encode('ascii') + file_path = input_dir / f'1294797712_1294797712_114.sub' with open(file_path, 'wb') as file: file.write(metadata) file.write(metadata_padding) file.write(zero_block) for _ in range(160): file.write(data_block2) - metadata = input_file_metadata_template.format(116,3).encode('ascii') - file_path = input_dir / f'1294797712_1294797712_116.sub' + metadata = input_file_metadata_template.format(116,3).encode('ascii') + file_path = input_dir / f'1294797712_1294797712_116.sub' with open(file_path, 'wb') as file: file.write(metadata) file.write(metadata_padding) file.write(zero_block) for _ in range(160): file.write(data_block3) - metadata = input_file_metadata_template.format(118,4).encode('ascii') - file_path = input_dir / f'1294797712_1294797712_118.sub' + metadata = input_file_metadata_template.format(118,4).encode('ascii') + file_path = input_dir / f'1294797712_1294797712_118.sub' with open(file_path, 'wb') as file: file.write(metadata) file.write(metadata_padding) file.write(zero_block) for _ in range(160): - file.write(data_block4) + file.write(data_block4) output_dir = working_dir / 'output_dir' output_dir.mkdir(exist_ok=False, parents=True) @@ -101,12 +101,13 @@ def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: assert set(output_dir_contents) == set(tile_output_filenames + [log_file_name]) # Note: need to manually inspect output log file. - + # Calculate Block + known = numpy.fft.irfft([0,0,2+29j,0,-83-8j,54,-160], norm=None)*12 for filename in tile_output_filenames: signal = read_output_signal(output_dir / filename) - expected = [42,255,115,0,167,255,246,255,188,255,118,1,82,254,46,1,59,255,246,255,39,0,187,0]*10240000 - signallist = signal.tolist() - assert signallist == expected + signal = numpy.reshape(signal, (160 * 64000,12)) + for row in signal : + numpy.array_equal(row, known) del signal def test_invalid_command_line_arguments(run_script: Path, working_dir: Path) -> None: @@ -309,7 +310,7 @@ def test_all_one_pfb(run_script: Path, working_dir: Path) -> None: inv_polyphase_filter_path = working_dir / 'inverse_polyphase_filter_2.bin' inv_polyphase_filter = numpy.zeros((13, 256), dtype=numpy.float32) write_inv_polyphase_filter(inv_polyphase_filter_path, inv_polyphase_filter) - + input_dir = working_dir / 'input_data' inverse_polyphase_filter_file = working_dir / 'inverse_polyphase_filter_2.bin' input_dir.mkdir(exist_ok=False, parents=True) @@ -412,7 +413,7 @@ def test_zero_signal_identity_ipfb(run_script: Path, working_dir: Path) -> None: output_dir.mkdir(exist_ok=False, parents=True) result = run_application(run_script, input_dir, '1294797712', '1294797712', inv_polyphase_filter_path, output_dir, 'false') - + assert result.returncode == 0 # Note: these tiles are flagged as faulty and will be skipped: 102, 115, 151, 164, 999, 2013, 2017, 2044, 2047 From 3556137c06ea4c595aef670d937a9b1250131609 Mon Sep 17 00:00:00 2001 From: Benjamin Bowers Date: Wed, 20 Oct 2021 21:12:43 +0800 Subject: [PATCH 19/19] Added accuracy check --- test/integration/test_main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/integration/test_main.py b/test/integration/test_main.py index d49a5e7..523367f 100644 --- a/test/integration/test_main.py +++ b/test/integration/test_main.py @@ -103,11 +103,15 @@ def test_many_signal_one_pfb(run_script: Path, working_dir: Path) -> None: # Note: need to manually inspect output log file. # Calculate Block known = numpy.fft.irfft([0,0,2+29j,0,-83-8j,54,-160], norm=None)*12 + timeDomain = numpy.full((160*64000, 12), known) + timeDomain = timeDomain.flatten() for filename in tile_output_filenames: signal = read_output_signal(output_dir / filename) - signal = numpy.reshape(signal, (160 * 64000,12)) - for row in signal : - numpy.array_equal(row, known) + errors = signal - timeDomain + norm_sq_errors = (errors * errors) / (timeDomain * timeDomain) + assert numpy.mean(norm_sq_errors) <= 0.005 + deviation_prob = numpy.mean(numpy.abs(errors) > numpy.std(timeDomain)) + assert deviation_prob <= 1e-6 del signal def test_invalid_command_line_arguments(run_script: Path, working_dir: Path) -> None: