From eccc8d9da8b6184f1c2fb0fa5a52451e5acc1202 Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Thu, 15 Aug 2024 14:24:10 +0200 Subject: [PATCH 1/7] adding support for handling merci3 data by renaming and extending the merci2 script --- README.md | 4 +- bin/{mersi22pps.py => mersi2pps.py} | 8 +- level1c4pps/__init__.py | 2 +- .../{mersi22pps_lib.py => mersi2pps_lib.py} | 105 +++++++++--------- level1c4pps/tests/__init__.py | 4 +- .../{test_mersi22pps.py => test_mersi2pps.py} | 50 +++------ 6 files changed, 79 insertions(+), 94 deletions(-) rename bin/{mersi22pps.py => mersi2pps.py} (89%) rename level1c4pps/{mersi22pps_lib.py => mersi2pps_lib.py} (53%) rename level1c4pps/tests/{test_mersi22pps.py => test_mersi2pps.py} (71%) diff --git a/README.md b/README.md index b9ba28d..95046c2 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,11 @@ agency specific level-1 formats. So far, supports - EUMETSAT Meteosat Second Generation SEVIRI HRIT level-1.5 - NOAA AVHRR GAC - - MERSI-2 level-1 + - MERSI-2/3 level-1 - MODIS level-1 - AVHRR eps level-1b - AVHRR AAPP level-1b - VIIRS level-1b - SLSTR level-1b - EPS-SG MetImage level-1 test data - - EUMETSAT AVHRR GAC FDR \ No newline at end of file + - EUMETSAT AVHRR GAC FDR diff --git a/bin/mersi22pps.py b/bin/mersi2pps.py similarity index 89% rename from bin/mersi22pps.py rename to bin/mersi2pps.py index 41bef86..4cc14b2 100755 --- a/bin/mersi22pps.py +++ b/bin/mersi2pps.py @@ -25,17 +25,17 @@ """Script to convert MERSI-2 level-1 to PPS level-1c format using Pytroll/Satpy.""" import argparse -from level1c4pps.mersi22pps_lib import process_one_scene +from level1c4pps.mersi2pps_lib import process_one_scene if __name__ == "__main__": """ Create PPS-format level1c data - From a list of MERSI-2 level-1 files create a NWCSAF/PPS formatet level1c file for pps. + From a list of MERSI-2/3 level-1 files create a NWCSAF/PPS formatet level1c file for pps. """ parser = argparse.ArgumentParser( - description=('Script to produce a PPS-level1c file for a MERSI-2 level-1 scene')) + description=('Script to produce a PPS-level1c file for a MERSI-2/3 level-1 scene')) parser.add_argument('files', metavar='fileN', type=str, nargs='+', - help='List of MERSI-2 files to process') + help='List of MERSI-2/3 files to process') parser.add_argument('-o', '--out_dir', type=str, nargs='?', required=False, default='.', help="Output directory where to store the level1c file") diff --git a/level1c4pps/__init__.py b/level1c4pps/__init__.py index 6cec4bd..0ae3963 100644 --- a/level1c4pps/__init__.py +++ b/level1c4pps/__init__.py @@ -184,7 +184,7 @@ def convert_angles(scene, delete_azimuth=False): 'sunazimuth': np.array([-18000, 18000], dtype='int16'), 'satazimuth': np.array([-18000, 18000], dtype='int16'), }, - 'mersi2_file_key': { + 'mersi_file_key': { 'sunzenith': 'Geolocation/SolarZenithAngle', 'satzenith': 'Geolocation/SensorZenithAngle', 'azimuthdiff': 'Geolocation/SensorSolarAzimuthDifference', diff --git a/level1c4pps/mersi22pps_lib.py b/level1c4pps/mersi2pps_lib.py similarity index 53% rename from level1c4pps/mersi22pps_lib.py rename to level1c4pps/mersi2pps_lib.py index 4a60f0f..9a5f9e4 100644 --- a/level1c4pps/mersi22pps_lib.py +++ b/level1c4pps/mersi2pps_lib.py @@ -22,18 +22,22 @@ # Nina Hakansson # Adam.Dybbroe -"""Functions to convert MERSI-2 level-1 data to a NWCSAF/PPS level-1c formatet netCDF/CF file.""" +"""Functions to convert MERSI-2/3 level-1 data to a NWCSAF/PPS level-1c formated netCDF/CF file.""" +import logging import os import time + +import numpy as np from satpy.scene import Scene -from level1c4pps import (get_encoding, compose_filename, +from level1c4pps import (ANGLE_ATTRIBUTES, + compose_filename, + convert_angles, + get_encoding, + get_header_attrs, + rename_latitude_longitude, set_header_and_band_attrs_defaults, - ANGLE_ATTRIBUTES, rename_latitude_longitude, - update_angle_attributes, get_header_attrs, - convert_angles) -import pyspectral # testing that pyspectral is available # noqa: F401 -import logging + update_angle_attributes) # Example: # tf2019234102243.FY3D-X_MERSI_GEOQK_L1B.HDF @@ -42,10 +46,15 @@ # tf2019234102243.FY3D-X_MERSI_0250M_L1B.HDF # -logger = logging.getLogger('mersi22pps') +logger = logging.getLogger('mersi2pps') + +SENSOR = {'FY3D': 'merci-2', + 'FY3F': 'merci-3', + 'FY3H': 'merci-3'} + +SATPY_READER = {'merci-2': 'mersi2_l1b', + 'merci-3': 'mersi3_l1b'} -PLATFORM_SHORTNAMES = {'FY3D': 'FY-3D'} -# BANDNAMES = ['%d' % (chn+1) for chn in range(25)] BANDNAMES = ['3', '4', '5', '6', '20', '22', '23', '24', '25'] REFL_BANDS = ['3', '4', '5', '6'] @@ -63,28 +72,22 @@ '24': 'ch_tb11', '25': 'ch_tb12'} -MERSI2_LEVEL1_FILE_PATTERN = 'tf{start_time:%Y%j%H%M%S}.{platform_shortname:4s}-X_MERSI_{dataset}_L1B.HDF' - - -def get_encoding_mersi2(scene): - """Get netcdf encoding for all datasets.""" - return get_encoding(scene, - BANDNAMES, - PPS_TAGNAMES, - chunks=None) - def set_header_and_band_attrs(scene, orbit_n=0): """Set and delete some attributes.""" irch = scene['24'] - nimg = set_header_and_band_attrs_defaults(scene, BANDNAMES, PPS_TAGNAMES, REFL_BANDS, irch, orbit_n=orbit_n) - scene.attrs['source'] = "mersi22pps.py" + nimg = set_header_and_band_attrs_defaults(scene, + BANDNAMES, + PPS_TAGNAMES, + REFL_BANDS, + irch, + orbit_n=orbit_n) + scene.attrs['source'] = "mersi2pps.py" return nimg def remove_broken_data(scene): """Set bad data to nodata.""" - import numpy as np for band in BANDNAMES: if band in REFL_BANDS: continue @@ -93,42 +96,40 @@ def remove_broken_data(scene): scene[band].values = scene[band].values + remove +def get_sensor(scene_file): + """Get sensor associated to the scene file.""" + for platform, sensor in SENSOR.items(): + if platform in scene_file: + return sensor + logger.info("Failed to determine sensor associated to scene file: '%s'", scene_file) + return None + + def process_one_scene(scene_files, out_path, engine='h5netcdf', orbit_n=0): """Make level 1c files in PPS-format.""" tic = time.time() - scn_ = Scene( - reader='mersi2_l1b', - filenames=scene_files) - - scn_.load(BANDNAMES + ['latitude', 'longitude'] + ANGLE_NAMES, resolution=1000) - - # Remove bad data at first and last column - remove_broken_data(scn_) - - # one ir channel - irch = scn_['24'] - - # Set header and band attributes - set_header_and_band_attrs(scn_, orbit_n=orbit_n) - - # Rename longitude, latitude to lon, lat. - rename_latitude_longitude(scn_) - - # Convert angles to PPS - convert_angles(scn_, delete_azimuth=True) - update_angle_attributes(scn_, irch) + sensor = get_sensor(os.path.basename(scene_files[0])) + reader = SATPY_READER[sensor] + scene = Scene(reader=reader, filenames=scene_files) + scene.load(BANDNAMES + ['latitude', 'longitude'] + ANGLE_NAMES, resolution=1000) + remove_broken_data(scene) + irch = scene['24'] # one ir channel + set_header_and_band_attrs(scene, orbit_n=orbit_n) + rename_latitude_longitude(scene) + convert_angles(scene, delete_azimuth=True) + update_angle_attributes(scene, irch) for angle in ['sunzenith', 'satzenith', 'azimuthdiff']: - scn_[angle].attrs['file_key'] = ANGLE_ATTRIBUTES['mersi2_file_key'][angle] + scene[angle].attrs['file_key'] = ANGLE_ATTRIBUTES['mersi_file_key'][angle] - filename = compose_filename(scn_, out_path, instrument='mersi2', band=irch) - scn_.save_datasets(writer='cf', + filename = compose_filename(scene, out_path, instrument=sensor.replace('-', ''), band=irch) + encoding = get_encoding(scene, BANDNAMES, PPS_TAGNAMES, chunks=None) + attrs = get_header_attrs(scene, band=irch, sensor=sensor) + scene.save_datasets(writer='cf', filename=filename, - header_attrs=get_header_attrs(scn_, band=irch, sensor='mersi-2'), + header_attrs=attrs, engine=engine, include_lonlats=False, flatten_attrs=True, - encoding=get_encoding_mersi2(scn_)) - print("Saved file {:s} after {:3.1f} seconds".format( - os.path.basename(filename), - time.time()-tic)) + encoding=encoding) + print(f"Saved file {os.path.basename(filename)} after {time.time() - tic:3.1f} seconds") return filename diff --git a/level1c4pps/tests/__init__.py b/level1c4pps/tests/__init__.py index e664ab7..1de273b 100644 --- a/level1c4pps/tests/__init__.py +++ b/level1c4pps/tests/__init__.py @@ -25,7 +25,7 @@ import unittest from level1c4pps.tests import (test_angles, test_seviri2pps, test_gac2pps, - test_mersi22pps, test_modis2pps, test_slstr2pps, + test_mersi2pps, test_modis2pps, test_slstr2pps, test_viirs2pps, test_eumgacfdr2pps, test_avhrr2pps, test_init) @@ -37,7 +37,7 @@ def suite(): mysuite.addTests(test_seviri2pps.suite()) mysuite.addTests(test_gac2pps.suite()) mysuite.addTests(test_eumgacfdr2pps.suite()) - mysuite.addTests(test_mersi22pps.suite()) + mysuite.addTests(test_mersi2pps.suite()) mysuite.addTests(test_modis2pps.suite()) mysuite.addTests(test_avhrr2pps.suite()) mysuite.addTests(test_slstr2pps.suite()) diff --git a/level1c4pps/tests/test_mersi22pps.py b/level1c4pps/tests/test_mersi2pps.py similarity index 71% rename from level1c4pps/tests/test_mersi22pps.py rename to level1c4pps/tests/test_mersi2pps.py index f96c738..9ed106d 100644 --- a/level1c4pps/tests/test_mersi22pps.py +++ b/level1c4pps/tests/test_mersi2pps.py @@ -21,7 +21,7 @@ # Stephan Finkensieper # Nina Hakansson -"""Unit tests for the gac2pps_lib module.""" +"""Unit tests for the merci2pps_lib module.""" import datetime as dt import unittest @@ -31,15 +31,15 @@ import mock from satpy import Scene -import level1c4pps.mersi22pps_lib as mersi22pps +import level1c4pps.mersi2pps_lib as mersi2pps -class TestMersi22PPS(unittest.TestCase): - """Test mersi22pps_lib.""" +class TestMersi2PPS(unittest.TestCase): + """Test mersi2pps_lib.""" def setUp(self): """Create a test scene.""" - mersi22pps.BANDNAMES = ['3', '24'] + mersi2pps.BANDNAMES = ['3', '24'] vis006 = mock.MagicMock(attrs={'name': 'image0', 'wavelength': [1, 2, 3, 'um'], 'id_tag': 'ch_r06'}) @@ -60,32 +60,6 @@ def setUp(self): self.scene[key].attrs['name'] = pps_name self.scene.attrs['sensor'] = ['mersi2'] - def test_get_encoding(self): - """Test encoding for MERSI-2.""" - enc_exp_angles = {'dtype': 'int16', - 'scale_factor': 0.01, - 'zlib': True, - 'complevel': 4, - '_FillValue': -32767, - 'add_offset': 0.0} - encoding_exp = { - 'image0': {'dtype': 'int16', - 'scale_factor': 0.01, - 'zlib': True, - 'complevel': 4, - '_FillValue': -32767, - 'add_offset': 0.0}, - 'image1': {'dtype': 'int16', - 'scale_factor': 0.01, - '_FillValue': -32767, - 'zlib': True, - 'complevel': 4, - 'add_offset': 273.15}, - 'satzenith': enc_exp_angles - } - encoding = mersi22pps.get_encoding_mersi2(self.scene) - self.assertDictEqual(encoding, encoding_exp) - def test_compose_filename(self): """Test compose filename for MERSI-2.""" start_time = dt.datetime(2009, 7, 1, 12, 15) @@ -99,15 +73,25 @@ def test_compose_filename(self): band = mock.MagicMock(attrs={'start_time': start_time, 'end_time': end_time}) fname_exp = '/out/path/S_NWC_mersi2_noaa19_99999_20090701T1216000Z_20090701T1227000Z.nc' - fname = mersi22pps.compose_filename(scene, '/out/path', 'mersi2', band=band) + fname = mersi2pps.compose_filename(scene, '/out/path', 'mersi2', band=band) self.assertEqual(fname, fname_exp) def test_set_header_and_band_attrs(self): """Test to set header_and_band_attrs.""" - mersi22pps.set_header_and_band_attrs(self.scene, orbit_n='12345') + mersi2pps.set_header_and_band_attrs(self.scene, orbit_n='12345') self.assertTrue(isinstance(self.scene.attrs['orbit_number'], int)) self.assertEqual(self.scene.attrs['orbit_number'], 12345) + def test_get_sensor(self): + """Test get sensor.""" + sensor = mersi2pps.get_sensor('tf2019234102243.FY3D-X_MERSI_GEOQK_L1B.HDF') + self.assertEqual(sensor, "merci-2") + + def test_get_sensor_returns_none(self): + """Test get sensor returns none for not recognized file.""" + sensor = mersi2pps.get_sensor('not_recognized_file') + self.assertEqual(sensor, None) + def suite(): """Create the test suite for test_mersi22pps.""" From c3c8c6721fc565ea0735f3da4f0e1801f00012bc Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Thu, 15 Aug 2024 15:21:10 +0200 Subject: [PATCH 2/7] changing path to script --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ddee0ce..803ac7e 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ packages=find_packages(), scripts=['bin/seviri2pps.py', 'bin/gac2pps.py', - 'bin/mersi22pps.py', + 'bin/mersi2pps.py', 'bin/viirs2pps.py', 'bin/vgac2pps.py', 'bin/slstr2pps.py', From 24a6c920cd902fe587780d889260630fb658858a Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Thu, 15 Aug 2024 16:25:32 +0200 Subject: [PATCH 3/7] fix spelling and some minor simplifications --- level1c4pps/mersi2pps_lib.py | 89 +++++++++++++++-------------- level1c4pps/tests/test_mersi2pps.py | 6 +- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/level1c4pps/mersi2pps_lib.py b/level1c4pps/mersi2pps_lib.py index 9a5f9e4..bcfd258 100644 --- a/level1c4pps/mersi2pps_lib.py +++ b/level1c4pps/mersi2pps_lib.py @@ -39,7 +39,7 @@ set_header_and_band_attrs_defaults, update_angle_attributes) -# Example: +# Example filenames: # tf2019234102243.FY3D-X_MERSI_GEOQK_L1B.HDF # tf2019234102243.FY3D-X_MERSI_GEO1K_L1B.HDF # tf2019234102243.FY3D-X_MERSI_1000M_L1B.HDF @@ -48,47 +48,51 @@ logger = logging.getLogger('mersi2pps') -SENSOR = {'FY3D': 'merci-2', - 'FY3F': 'merci-3', - 'FY3H': 'merci-3'} +SENSOR = {'FY3D': 'mersi-2', + 'FY3F': 'mersi-3', + 'FY3H': 'mersi-3'} -SATPY_READER = {'merci-2': 'mersi2_l1b', - 'merci-3': 'mersi3_l1b'} - -BANDNAMES = ['3', '4', '5', '6', '20', '22', '23', '24', '25'] +SATPY_READER = {'mersi-2': 'mersi2_l1b', + 'mersi-3': 'mersi3_l1b'} REFL_BANDS = ['3', '4', '5', '6'] -ANGLE_NAMES = ['satellite_zenith_angle', 'solar_zenith_angle', - 'satellite_azimuth_angle', 'solar_azimuth_angle'] +ONE_IR_CHANNEL = '24' + +GEOLOCATION_NAMES = ['latitude', + 'longitude', + 'satellite_azimuth_angle', + 'satellite_zenith_angle', + 'solar_azimuth_angle', + 'solar_zenith_angle'] + +PPS_BAND_NAME = {'3': 'ch_r06', + '4': 'ch_r09', + '5': 'ch_r13', + '6': 'ch_r16', + '20': 'ch_tb37', + '22': 'ch_tb73', + '23': 'ch_tb85', + '24': 'ch_tb11', + '25': 'ch_tb12'} -PPS_TAGNAMES = {'3': 'ch_r06', - '4': 'ch_r09', - '5': 'ch_r13', - '6': 'ch_r16', - '20': 'ch_tb37', - '22': 'ch_tb73', - '23': 'ch_tb85', - '24': 'ch_tb11', - '25': 'ch_tb12'} +RESOLUTION = 1000 -def set_header_and_band_attrs(scene, orbit_n=0): +def set_header_and_band_attrs(scene, band, orbit_n): """Set and delete some attributes.""" - irch = scene['24'] - nimg = set_header_and_band_attrs_defaults(scene, - BANDNAMES, - PPS_TAGNAMES, - REFL_BANDS, - irch, - orbit_n=orbit_n) + set_header_and_band_attrs_defaults(scene, + list(PPS_BAND_NAME), + PPS_BAND_NAME, + REFL_BANDS, + band, + orbit_n=orbit_n) scene.attrs['source'] = "mersi2pps.py" - return nimg def remove_broken_data(scene): """Set bad data to nodata.""" - for band in BANDNAMES: + for band in PPS_BAND_NAME: if band in REFL_BANDS: continue if band in scene: @@ -111,25 +115,26 @@ def process_one_scene(scene_files, out_path, engine='h5netcdf', orbit_n=0): sensor = get_sensor(os.path.basename(scene_files[0])) reader = SATPY_READER[sensor] scene = Scene(reader=reader, filenames=scene_files) - scene.load(BANDNAMES + ['latitude', 'longitude'] + ANGLE_NAMES, resolution=1000) + band_names = list(PPS_BAND_NAME) + scene.load(band_names + GEOLOCATION_NAMES, resolution=RESOLUTION) remove_broken_data(scene) - irch = scene['24'] # one ir channel - set_header_and_band_attrs(scene, orbit_n=orbit_n) + band = scene[ONE_IR_CHANNEL] + set_header_and_band_attrs(scene, band, orbit_n) rename_latitude_longitude(scene) convert_angles(scene, delete_azimuth=True) - update_angle_attributes(scene, irch) + update_angle_attributes(scene, band) for angle in ['sunzenith', 'satzenith', 'azimuthdiff']: scene[angle].attrs['file_key'] = ANGLE_ATTRIBUTES['mersi_file_key'][angle] - filename = compose_filename(scene, out_path, instrument=sensor.replace('-', ''), band=irch) - encoding = get_encoding(scene, BANDNAMES, PPS_TAGNAMES, chunks=None) - attrs = get_header_attrs(scene, band=irch, sensor=sensor) + filename = compose_filename(scene, out_path, instrument=sensor.replace('-', ''), band=band) + encoding = get_encoding(scene, band_names, PPS_BAND_NAME, chunks=None) + attrs = get_header_attrs(scene, band=band, sensor=sensor) scene.save_datasets(writer='cf', - filename=filename, - header_attrs=attrs, - engine=engine, - include_lonlats=False, - flatten_attrs=True, - encoding=encoding) + filename=filename, + header_attrs=attrs, + engine=engine, + include_lonlats=False, + flatten_attrs=True, + encoding=encoding) print(f"Saved file {os.path.basename(filename)} after {time.time() - tic:3.1f} seconds") return filename diff --git a/level1c4pps/tests/test_mersi2pps.py b/level1c4pps/tests/test_mersi2pps.py index 9ed106d..e51348c 100644 --- a/level1c4pps/tests/test_mersi2pps.py +++ b/level1c4pps/tests/test_mersi2pps.py @@ -78,14 +78,16 @@ def test_compose_filename(self): def test_set_header_and_band_attrs(self): """Test to set header_and_band_attrs.""" - mersi2pps.set_header_and_band_attrs(self.scene, orbit_n='12345') + mersi2pps.set_header_and_band_attrs(self.scene, band=self.scene['24'], orbit_n='12345') self.assertTrue(isinstance(self.scene.attrs['orbit_number'], int)) self.assertEqual(self.scene.attrs['orbit_number'], 12345) def test_get_sensor(self): """Test get sensor.""" sensor = mersi2pps.get_sensor('tf2019234102243.FY3D-X_MERSI_GEOQK_L1B.HDF') - self.assertEqual(sensor, "merci-2") + self.assertEqual(sensor, "mersi-2") + sensor = mersi2pps.get_sensor('tf2019234102243.FY3F-X_MERSI_GEOQK_L1B.HDF') + self.assertEqual(sensor, "mersi-3") def test_get_sensor_returns_none(self): """Test get sensor returns none for not recognized file.""" From e40df726e52313c0b8fa67c297de497443bdfbfa Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Fri, 16 Aug 2024 10:54:17 +0200 Subject: [PATCH 4/7] fixing and adding test --- level1c4pps/mersi2pps_lib.py | 113 ++++++++++++++-------------- level1c4pps/tests/test_mersi2pps.py | 40 +++++++--- 2 files changed, 86 insertions(+), 67 deletions(-) diff --git a/level1c4pps/mersi2pps_lib.py b/level1c4pps/mersi2pps_lib.py index bcfd258..253ea6c 100644 --- a/level1c4pps/mersi2pps_lib.py +++ b/level1c4pps/mersi2pps_lib.py @@ -21,6 +21,7 @@ # Martin Raspaud # Nina Hakansson # Adam.Dybbroe +# Bengt.Rydberg >bengt.rydberg@smhi.se> """Functions to convert MERSI-2/3 level-1 data to a NWCSAF/PPS level-1c formated netCDF/CF file.""" @@ -30,14 +31,16 @@ import numpy as np from satpy.scene import Scene -from level1c4pps import (ANGLE_ATTRIBUTES, - compose_filename, - convert_angles, - get_encoding, - get_header_attrs, - rename_latitude_longitude, - set_header_and_band_attrs_defaults, - update_angle_attributes) +from level1c4pps import ( + ANGLE_ATTRIBUTES, + compose_filename, + convert_angles, + get_encoding, + get_header_attrs, + rename_latitude_longitude, + set_header_and_band_attrs_defaults, + update_angle_attributes, +) # Example filenames: # tf2019234102243.FY3D-X_MERSI_GEOQK_L1B.HDF @@ -48,56 +51,53 @@ logger = logging.getLogger('mersi2pps') -SENSOR = {'FY3D': 'mersi-2', - 'FY3F': 'mersi-3', - 'FY3H': 'mersi-3'} - -SATPY_READER = {'mersi-2': 'mersi2_l1b', - 'mersi-3': 'mersi3_l1b'} - +SENSOR = { # sensor associated to platform + 'FY3D': 'mersi-2', + 'FY3F': 'mersi-3', + 'FY3H': 'mersi-3', +} +SATPY_READER = { # satpy reader associated to sensor + 'mersi-2': 'mersi2_l1b', + 'mersi-3': 'mersi3_l1b', +} +PPS_BAND_NAME = { # PPS band name associated to satpy band name + '3': 'ch_r06', + '4': 'ch_r09', + '5': 'ch_r13', + '6': 'ch_r16', + '20': 'ch_tb37', + '22': 'ch_tb73', + '23': 'ch_tb85', + '24': 'ch_tb11', + '25': 'ch_tb12', +} +GEOLOCATION_NAMES = [ # additional variables to load + 'latitude', + 'longitude', + 'satellite_azimuth_angle', + 'satellite_zenith_angle', + 'solar_azimuth_angle', + 'solar_zenith_angle', +] REFL_BANDS = ['3', '4', '5', '6'] - ONE_IR_CHANNEL = '24' - -GEOLOCATION_NAMES = ['latitude', - 'longitude', - 'satellite_azimuth_angle', - 'satellite_zenith_angle', - 'solar_azimuth_angle', - 'solar_zenith_angle'] - -PPS_BAND_NAME = {'3': 'ch_r06', - '4': 'ch_r09', - '5': 'ch_r13', - '6': 'ch_r16', - '20': 'ch_tb37', - '22': 'ch_tb73', - '23': 'ch_tb85', - '24': 'ch_tb11', - '25': 'ch_tb12'} - -RESOLUTION = 1000 +RESOLUTION = 1000 # [m] +LOW_TB = 1 # [K] very cold brightness temperature def set_header_and_band_attrs(scene, band, orbit_n): """Set and delete some attributes.""" - set_header_and_band_attrs_defaults(scene, - list(PPS_BAND_NAME), - PPS_BAND_NAME, - REFL_BANDS, - band, - orbit_n=orbit_n) + set_header_and_band_attrs_defaults( + scene, list(PPS_BAND_NAME), PPS_BAND_NAME, REFL_BANDS, band, orbit_n=orbit_n, + ) scene.attrs['source'] = "mersi2pps.py" def remove_broken_data(scene): """Set bad data to nodata.""" for band in PPS_BAND_NAME: - if band in REFL_BANDS: - continue - if band in scene: - remove = np.where(scene[band].values < 1, np.nan, 0) # 1K very cold - scene[band].values = scene[band].values + remove + if band in band not in REFL_BANDS and band in scene: + scene[band].data = np.where(scene[band].data < LOW_TB, np.nan, scene[band].data) def get_sensor(scene_file): @@ -125,16 +125,15 @@ def process_one_scene(scene_files, out_path, engine='h5netcdf', orbit_n=0): update_angle_attributes(scene, band) for angle in ['sunzenith', 'satzenith', 'azimuthdiff']: scene[angle].attrs['file_key'] = ANGLE_ATTRIBUTES['mersi_file_key'][angle] - - filename = compose_filename(scene, out_path, instrument=sensor.replace('-', ''), band=band) - encoding = get_encoding(scene, band_names, PPS_BAND_NAME, chunks=None) - attrs = get_header_attrs(scene, band=band, sensor=sensor) - scene.save_datasets(writer='cf', - filename=filename, - header_attrs=attrs, - engine=engine, - include_lonlats=False, - flatten_attrs=True, - encoding=encoding) + filename=compose_filename(scene, out_path, instrument=sensor.replace('-', ''), band=band) + scene.save_datasets( + writer='cf', + filename=filename, + header_attrs=get_header_attrs(scene, band=band, sensor=sensor), + engine=engine, + include_lonlats=False, + flatten_attrs=True, + encoding=get_encoding(scene, band_names, PPS_BAND_NAME, chunks=None), + ) print(f"Saved file {os.path.basename(filename)} after {time.time() - tic:3.1f} seconds") return filename diff --git a/level1c4pps/tests/test_mersi2pps.py b/level1c4pps/tests/test_mersi2pps.py index e51348c..d526143 100644 --- a/level1c4pps/tests/test_mersi2pps.py +++ b/level1c4pps/tests/test_mersi2pps.py @@ -29,6 +29,8 @@ from unittest import mock except ImportError: import mock +import numpy as np +import xarray as xr from satpy import Scene import level1c4pps.mersi2pps_lib as mersi2pps @@ -82,23 +84,41 @@ def test_set_header_and_band_attrs(self): self.assertTrue(isinstance(self.scene.attrs['orbit_number'], int)) self.assertEqual(self.scene.attrs['orbit_number'], 12345) + def test_remove_broken_data(self): + """Test remove broken data.""" + data = xr.Dataset( + { + '3': (('y', 'x'), [[100, 0, 100, 100]]), + '20': (('y', 'x'), [[200, 0, 200, 200]]), + '22': (('y', 'x'), [[300, 0, 300, 300]]), + } + ) + mersi2pps.remove_broken_data(data) + expect = xr.Dataset( + { + '3': (('y', 'x'), [[100, 0, 100, 100]]), + '20': (('y', 'x'), [[200, np.nan, 200, 200]]), + '22': (('y', 'x'), [[300, np.nan, 300, 300]]), + } + ) + for band in expect: + np.testing.assert_array_equal(data[band], expect[band]) + def test_get_sensor(self): """Test get sensor.""" - sensor = mersi2pps.get_sensor('tf2019234102243.FY3D-X_MERSI_GEOQK_L1B.HDF') - self.assertEqual(sensor, "mersi-2") - sensor = mersi2pps.get_sensor('tf2019234102243.FY3F-X_MERSI_GEOQK_L1B.HDF') - self.assertEqual(sensor, "mersi-3") - - def test_get_sensor_returns_none(self): - """Test get sensor returns none for not recognized file.""" - sensor = mersi2pps.get_sensor('not_recognized_file') - self.assertEqual(sensor, None) + for filename, expect in [ + ('tf2019234102243.FY3D-X_MERSI_GEOQK_L1B.HDF', 'mersi-2'), + ('tf2019234102243.FY3F-X_MERSI_GEOQK_L1B.HDF', 'mersi-3'), + ('not_recognized_file', None), + ]: + sensor = mersi2pps.get_sensor(filename) + self.assertEqual(sensor, expect) def suite(): """Create the test suite for test_mersi22pps.""" loader = unittest.TestLoader() mysuite = unittest.TestSuite() - mysuite.addTest(loader.loadTestsFromTestCase(TestMersi22PPS)) + mysuite.addTest(loader.loadTestsFromTestCase(TestMersi2PPS)) return mysuite From fe4721d776009a9e6c8421801f570b19285f2693 Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Fri, 16 Aug 2024 12:14:48 +0200 Subject: [PATCH 5/7] fixing typo --- level1c4pps/mersi2pps_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/level1c4pps/mersi2pps_lib.py b/level1c4pps/mersi2pps_lib.py index 253ea6c..366ac53 100644 --- a/level1c4pps/mersi2pps_lib.py +++ b/level1c4pps/mersi2pps_lib.py @@ -96,7 +96,7 @@ def set_header_and_band_attrs(scene, band, orbit_n): def remove_broken_data(scene): """Set bad data to nodata.""" for band in PPS_BAND_NAME: - if band in band not in REFL_BANDS and band in scene: + if band not in REFL_BANDS and band in scene: scene[band].data = np.where(scene[band].data < LOW_TB, np.nan, scene[band].data) From eb652c7ca7622ec3c3c4b9ed0c870257cc3d1759 Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Tue, 8 Oct 2024 13:17:36 +0200 Subject: [PATCH 6/7] adding script as a symbolici link --- bin/mersi22pps.py | 1 + setup.py | 4 ++++ 2 files changed, 5 insertions(+) create mode 120000 bin/mersi22pps.py diff --git a/bin/mersi22pps.py b/bin/mersi22pps.py new file mode 120000 index 0000000..75daaea --- /dev/null +++ b/bin/mersi22pps.py @@ -0,0 +1 @@ +mersi2pps.py \ No newline at end of file diff --git a/setup.py b/setup.py index 803ac7e..31a588c 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ scripts=['bin/seviri2pps.py', 'bin/gac2pps.py', 'bin/mersi2pps.py', + 'bin/mersi22pps.py', 'bin/viirs2pps.py', 'bin/vgac2pps.py', 'bin/slstr2pps.py', @@ -67,6 +68,9 @@ 'bin/eumgacfdr2pps.py', 'bin/modis2pps.py', 'bin/avhrr2pps.py'], + entry_points={ + 'console_scripts': ['mersi22pps.py=bin.mersi2pps.__main__:main'], + }, data_files=[], zip_safe=False, use_scm_version=True, From fad5ca0c2a4befa8cbc71f5baf05403977234993 Mon Sep 17 00:00:00 2001 From: BengtRydberg Date: Tue, 8 Oct 2024 13:22:14 +0200 Subject: [PATCH 7/7] remove leftovers --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index 31a588c..4764bb1 100644 --- a/setup.py +++ b/setup.py @@ -68,9 +68,6 @@ 'bin/eumgacfdr2pps.py', 'bin/modis2pps.py', 'bin/avhrr2pps.py'], - entry_points={ - 'console_scripts': ['mersi22pps.py=bin.mersi2pps.__main__:main'], - }, data_files=[], zip_safe=False, use_scm_version=True,