Merge pull request #77 from ninahakansson/cmsaf_vgac

VGAC converter

bin/vgac2pps.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 level1c4pps developers
+#
+# This file is part of level1c4pps
+#
+# level1c4pps is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# level1c4pps is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with level1c4pps.  If not, see <http://www.gnu.org/licenses/>.
+# Author(s):
+
+#   Martin Raspaud <[email protected]>
+#   Nina Hakansson <[email protected]>
+#   Adam.Dybbroe <[email protected]>
+
+"""Script to convert VIIRS level-1 to PPS level-1c format using Pytroll/Satpy."""
+
+import argparse
+from level1c4pps.vgac2pps_lib import process_one_scene
+
+
+if __name__ == "__main__":
+    """ Create PPS-format level1c data
+    From a list of VIIRS level-1 files create a NWCSAF/PPS formatet level1c file for pps.
+    """
+    parser = argparse.ArgumentParser(
+        description=('Script to produce a PPS-level1c file for a VIIRS level-1 scene'))
+    parser.add_argument('files', metavar='fileN', type=str, nargs='+',
+                        help='List of VIIRS files to process')
+    parser.add_argument('-o', '--out_dir', type=str, nargs='?',
+                        required=False, default='.',
+                        help="Output directory where to store the level1c file")
+    parser.add_argument('-ne', '--nc_engine', type=str, nargs='?',
+                        required=False, default='h5netcdf',
+                        help="Engine for saving netcdf files netcdf4 or h5netcdf (default).")
+    parser.add_argument('-all_ch', '--all_channels', action='store_true',
+                        help="Save all 21 channels to level1c4pps file.")
+    parser.add_argument('-n19', '--as_noaa19', action='store_true',
+                        help="Save only the AVHRR (1,2, 3B, 4, 5) channels to level1c4pps file. And apply SBAFs to the channels.")
+    parser.add_argument('-pps_ch', '--pps_channels', action='store_true',
+                        help="Save only the necessary (for PPS) channels to level1c4pps file.")
+    parser.add_argument('-avhrr_ch', '--avhrr_channels', action='store_true',
+                        help="Save only the AVHRR (1,2, 3B, 4, 5) channels to level1c4pps file.")
+    parser.add_argument('-on', '--orbit_number', type=int, nargs='?',
+                        required=False, default=0,
+                        help="Orbit number (default is 00000).")
+
+    options = parser.parse_args()
+    process_one_scene(options.files, options.out_dir, engine=options.nc_engine,
+                      all_channels=options.all_channels, pps_channels=options.pps_channels,
+                      orbit_n=options.orbit_number, as_noaa19=options.as_noaa19, avhrr_channels=options.avhrr_channels)

level1c4pps/__init__.py

@@ -130,14 +130,18 @@
 SATPY_ANGLE_NAMES = {
     'solar_zenith': 'sunzenith',  # no _angle
     'solar_zenith_angle': 'sunzenith',
+    'sza': 'sunzenith',
     'solar_azimuth': 'sunazimuth',  # no _angle
     'solar_azimuth_angle': 'sunazimuth',
+    'azn': 'sunazimuth',
     'satellite_zenith_angle': 'satzenith',
     'sensor_zenith_angle': 'satzenith',
     'observation_zenith': 'satzenith',
+    'vza': 'satzenith',
     'satellite_azimuth_angle': 'satazimuth',
     'sensor_azimuth_angle': 'satazimuth',
     'observation_azimuth': 'satazimuth',
+    'azi': 'satazimuth',
     'sun_sensor_azimuth_difference_angle': 'azimuthdiff',
 }
 
@@ -250,6 +254,10 @@ def dt64_to_datetime(dt64):
         seconds_since_epoch = (dt64 - unix_epoch) / one_second
         dt = datetime.utcfromtimestamp(seconds_since_epoch)
         return dt
+    elif type(dt64) == np.float64:
+        seconds_since_epoch = dt64
+        dt = datetime.utcfromtimestamp(seconds_since_epoch)
+        return dt  
     return dt64
 
 
@@ -520,6 +528,7 @@ def platform_name_to_use_in_filename(platform_name):
         new_name = 'metopsga1'
     replace_dict = {'aqua': '2',
                     '-': '',
+                    'jpss1': 'noaa20',
                     'terra': '1',
                     'suomi': ''}
     for orig, new in replace_dict.items():

level1c4pps/tests/test_vgac2pps.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 level1c4pps developers
+#
+# This file is part of level1c4pps
+#
+# level1c4pps is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# level1c4pps is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with level1c4pps.  If not, see <http://www.gnu.org/licenses/>.
+# Author(s):
+
+#   Nina Hakansson <[email protected]>
+
+"""Unit tests for the vgac2pps_lib module."""
+
+import netCDF4
+import unittest
+from datetime import datetime
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+from satpy import Scene
+
+import level1c4pps.vgac2pps_lib as vgac2pps
+import numpy as np
+
+
+class TestVgac2PPS(unittest.TestCase):
+    """Test vgac2pps_lib."""
+
+    def setUp(self):
+        """Create a test scene."""
+        vis006 = mock.MagicMock(attrs={'name': 'image0',
+                                       'wavelength': [1, 2, 3, 'um'],
+                                       'id_tag': 'ch_r06'})
+        ir_108 = mock.MagicMock(attrs={'name': 'image1',
+                                       'id_tag': 'ch_tb11',
+                                       'wavelength': [1, 2, 3, 'um'],
+                                       'start_time': datetime.utcnow(),
+                                       'end_time': datetime.utcnow(),
+                                       'history': 'dummy',
+                                       'platform_name': 'tirosn',
+                                       'orbit_number': 99999})
+        qual_f = mock.MagicMock(attrs={'name': 'qual_flags',
+                                       'id_tag': 'qual_flags'})
+        scan_t = mock.MagicMock(attrs={'name': 'scanline_timestamps'})
+        self.scene = Scene()
+        self.scene.attrs['sensor'] = ['avhrr-1', 'avhrr-2', 'avhrr-3']
+        scene_dict = {'M05': vis006, 'M15': ir_108,
+                      'qual_flags': qual_f, 'acq_time': scan_t}
+        for key in scene_dict:
+            pps_name = scene_dict[key].attrs['name']
+            self.scene[key] = scene_dict[key]
+            self.scene[key].attrs['name'] = pps_name
+
+    def test_get_encoding(self):
+        """Test the encoding for VGAC."""
+        encoding_exp = {
+            'image0': {'dtype': 'int16',
+                       'scale_factor': 0.01,
+                       'zlib': True,
+                       'complevel': 4,
+                       '_FillValue': -32767,
+                       'add_offset': 0.0},
+            'image1': {'dtype': 'int16',
+                       'scale_factor': 0.01,
+                       '_FillValue': -32767,
+                       'zlib': True,
+                       'complevel': 4,
+                       'add_offset': 273.15},
+            'qual_flags':  {'dtype': 'int16', 'zlib': True,
+                            'complevel': 4, '_FillValue': -32001.0},
+            'scanline_timestamps': {'dtype': 'int64', 'zlib': True,
+                                    'units': 'Milliseconds since 1970-01-01',
+                                    'complevel': 4, '_FillValue': -1.0},
+        }
+        encoding = vgac2pps.get_encoding_viirs(self.scene)
+        self.assertDictEqual(encoding, encoding_exp)
+
+    def test_set_header_and_band_attrs(self):
+        """Test to set header_and_band_attrs."""
+        vgac2pps.set_header_and_band_attrs(self.scene, orbit_n='12345')
+        self.assertEqual(self.scene.attrs['orbit_number'], 12345)
+
+    def test_process_one_scene(self):
+        """Test process one scene for one example file."""
+
+        vgac2pps.process_one_scene(
+            ['./level1c4pps/tests/VGAC_VJ102MOD_A2018305_1042_n004946_K005.nc'],
+            out_path='./level1c4pps/tests/',
+        )
+        filename = './level1c4pps/tests/S_NWC_viirs_noaa20_00000_20181101T1042080Z_20181101T1224090Z.nc'
+        # written with hfnetcdf read with NETCDF4 ensure compatability
+        pps_nc = netCDF4.Dataset(filename, 'r', format='NETCDF4')  # Check compatability implicitly
+
+        for key in ['start_time', 'end_time', 'history', 'instrument',
+                    'orbit_number', 'platform',
+                    'sensor', 'source']:
+            if key not in pps_nc.__dict__.keys():
+                print("Missing in attributes:", key)
+            self.assertTrue(key in pps_nc.__dict__.keys())
+
+        expected_vars = ['satzenith', 'azimuthdiff',
+                         'satazimuth', 'sunazimuth', 'sunzenith',
+                         'lon', 'lat',
+                         'image1', 'image2', 'image3', 'image4', 'image5',
+                         'image6', 'image7', 'image8', 'image9',
+                         'scanline_timestamps', 'time', 'time_bnds']
+        for var in expected_vars:
+            self.assertTrue(var in pps_nc.variables.keys())
+
+        np.testing.assert_almost_equal(pps_nc.variables['image1'].sun_earth_distance_correction_factor,
+                                       1.0, decimal=4)
+    def test_process_one_scene_n19(self):
+        """Test process one scene for one example file."""
+
+        vgac2pps.process_one_scene(
+            ['./level1c4pps/tests/VGAC_VJ102MOD_A2018305_1042_n004946_K005.nc'],
+            out_path='./level1c4pps/tests/',
+            as_noaa19=True,
+        )
+        filename = './level1c4pps/tests/S_NWC_avhrr_vgac20_00000_20181101T1042080Z_20181101T1224090Z.nc'
+        filename_viirs = './level1c4pps/tests/S_NWC_viirs_noaa20_00000_20181101T1042080Z_20181101T1224090Z.nc'
+        # written with hfnetcdf read with NETCDF4 ensure compatability
+        pps_nc = netCDF4.Dataset(filename, 'r', format='NETCDF4')  # Check compatability implicitly
+        pps_nc_viirs = netCDF4.Dataset(filename_viirs, 'r', format='NETCDF4')  # Check compatability implicitly
+
+        for key in ['start_time', 'end_time', 'history', 'instrument',
+                    'orbit_number', 'platform',
+                    'sensor', 'source']:
+            if key not in pps_nc.__dict__.keys():
+                print("Missing in attributes:", key)
+            self.assertTrue(key in pps_nc.__dict__.keys())
+
+        expected_vars = ['satzenith', 'azimuthdiff',
+                         'satazimuth', 'sunazimuth', 'sunzenith',
+                         'lon', 'lat',
+                         'image1', 'image2', 'image3', 'image4', 'image5',
+                         'scanline_timestamps', 'time', 'time_bnds']
+
+        for var in expected_vars:
+            self.assertTrue(var in pps_nc.variables.keys())
+
+        np.testing.assert_almost_equal(pps_nc.variables['image1'].sun_earth_distance_correction_factor,
+                                       1.0, decimal=4)
+
+        np.testing.assert_equal(pps_nc.__dict__["platform"], "vgac20")
+        self.assertTrue(np.abs(pps_nc.variables['image1'][0,0,0] - pps_nc_viirs.variables['image1'][0,0,0])>0.01)