Merge pull request #46 from GEOS-ESM/develop

v2.0.1

CHANGELOG.md

@@ -8,6 +8,21 @@
 
 ### Changed
 
+## [v2.0.1] - 2023-05-17
+
+### Added
+
+
+### Fixed
+
+- fix in trj_sampler for ICARTT files that have missing values in the location/time
+- fix in py_ods that was trying to do element wise comparisons on tuples and lists
+
+### Changed
+
+- updated GMAOpyobs to v1.0.2
+- updates to NNR training code to handle angstrom exponent targets
+- some other minor changes to NNR training code to be python3 compliant
 
 ## [v2.0.0] - 2023-05-17
 

CMakeLists.txt

@@ -4,7 +4,7 @@ cmake_policy (SET CMP0054 NEW)
 
 project (
   AeroApps
-  VERSION 2.0.0
+  VERSION 2.0.1
   LANGUAGES Fortran CXX C)  # Note - CXX is required for ESMF
 
 if ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")

components.yaml

@@ -28,5 +28,5 @@ MAPL:
 GMAOpyobs:
   local: ./src/Shared/GMAO_Shared/GMAO_pyobs@
   remote: ../GMAOpyobs.git
-  tag: v1.0.1
+  tag: v1.0.2
   develop: develop

src/Applications/GAAS_App/modis_l2a.py

@@ -11,7 +11,7 @@
 
 from os       import system
 from optparse import OptionParser
-from MAPL     import Config
+from MAPL.config     import Config
 
 if __name__ == "__main__":
 

src/Applications/GAAS_App/mxd04_l2a.py

@@ -21,11 +21,10 @@
 import sys
 import subprocess
 
-from time            import clock
 from optparse        import OptionParser   # Command-line args  
 from dateutil.parser import parse as isoparse
 from mxd04_nnr       import MxD04_NNR
-from MAPL            import strTemplate
+from MAPL.config     import strTemplate
 
 Ident = dict( modo = ('MOD04','ocean'),
               modl = ('MOD04','land'),
@@ -139,7 +138,6 @@ def makethis_dir(filename):
         print("                          MODIS Level 2A Processing")
         print("                          -------------------------")
         print("")
-        t0 = clock()
 
 #   Time variables
 #   --------------

src/Applications/GAAS_App/mxd04_nnr.py

@@ -99,6 +99,11 @@
                          aTau550 = ( 'aod_', 550 ),
                          aTau660 = ( 'aod_', 660 ),
                          aTau870 = ( 'aod_', 870 ),
+                         aAE440  = ( 'aod_', 440 ),
+                         aAE470  = ( 'aod_', 470 ),
+                         aAE500  = ( 'aod_', 500 ),
+                         aAE660  = ( 'aod_', 660 ),
+                         aAE870  = ( 'aod_', 870 ),
                          )
 
 class MxD04_NNR(MxD04_L2):
@@ -114,7 +119,9 @@ def __init__(self,l2_path,prod,algo,syn_time,aer_x,
                  scat_thresh=170.0,
                  cloudFree=None,
                  aodmax=1.0,
-                 coll='006',verbose=0):
+                 coll='006',
+                 nsyn=8,
+                 verbose=0):
         """
         Contructs a MXD04 object from MODIS Aerosol Level 2
         granules. On input,
@@ -129,7 +136,9 @@ def __init__(self,l2_path,prod,algo,syn_time,aer_x,
         cloud_tresh --- cloud fraction treshhold
         cloudFree   --- cloud fraction threshhold for assuring no cloud contaminations when aod is > aodmax
                         if None, no cloud free check is made
-              
+        coll         --- MODIS data collection
+        nsyn         --- number of synoptic times              
+
         The following attributes are also defined:
            fractions dust, sea salt, BC+OC, sulfate
            aod_coarse
@@ -148,15 +157,15 @@ def __init__(self,l2_path,prod,algo,syn_time,aer_x,
 
         # Initialize superclass
         # ---------------------
-        Files = granules(l2_path,prod,syn_time,coll=coll)
+        Files = granules(l2_path,prod,syn_time,coll=coll,nsyn=nsyn)
         if algo != "DEEP":
-            MxD04_L2.__init__(self,Files,algo,syn_time,
+            MxD04_L2.__init__(self,Files,algo,syn_time=syn_time,nsyn=nsyn,
                               only_good=True,
                               SDS=SDS,
                               alias={'Deep_Blue_Cloud_Fraction_Land':'cloud_deep'},
                               Verb=verbose)            
         else:        
-            MxD04_L2.__init__(self,Files,algo,syn_time,
+            MxD04_L2.__init__(self,Files,algo,syn_time=syn_time,nsyn=nsyn,
                               only_good=False,
                               SDS=SDS,                            
                               alias=ALIAS,
@@ -304,6 +313,14 @@ def speciate(self,aer_x,Verbose=False):
         except:
             pass   # ignore it for systems without nitrates
 
+        # Handle brown carbon
+        # --------------------
+        try:
+            self.sampleFile(aer_x,onlyVars=('BREXTTAU',),Verbose=Verbose)
+            self.fcc += self.sample.BRCEXTTAU / s.TOTEXTTAU
+        except:
+            pass   # ignore it for systems without brown carbon
+
         del self.sample
 
 #---
@@ -437,6 +454,32 @@ def apply(self,nnFile):
         # ---------------------
         targets = self.net(self._getInputs())
 
+        # If target is angstrom exponent
+        # calculate AOD 
+        # ------------------------------
+        doAE = False
+        for targetName in self.net.TargetNames:
+            if 'AE' in targetName:
+                doAE = True
+
+        if doAE:
+            for i,targetName in enumerate(self.net.TargetNames):
+                if 'Tau' in targetName:
+                    name, base_wav = TranslateTarget[targetName]
+                    base_wav = np.float(base_wav)
+                    base_tau = targets[:,i]
+                    if self.net.laod:
+                        base_tau = exp(base_tau) - 0.01 # inverse
+            for i,targetName in enumerate(self.net.TargetNames):
+                if 'AE' in targetName:
+                    AE = targets[:,i]
+                    name, wav = TranslateTarget[targetName]
+                    wav = np.float(wav)
+                    data = base_tau*np.exp(-1.*AE*np.log(wav/base_wav))
+                    if self.net.laod:
+                        targets[:,i] = np.log(data + 0.01)
+                    else:
+                        targets[:,i] = data
 
         # Targets do not have to be in MODIS retrieval
         # ----------------------------------------------

src/Components/misc/obs_aod/ABC/CMakeLists.txt

@@ -35,7 +35,6 @@ set (PYFILES
     evaluate_avhrr.py
     evaluate_cdr.py
     giant.py
-    man.py
     mcd43c.py
     nn.py
     reduce_avhrr.py

src/Components/misc/obs_aod/ABC/abc_c6.py

@@ -22,7 +22,7 @@
 from   sklearn.linear_model import LinearRegression
 from   multiprocessing      import cpu_count
 from   abc_c6_aux           import SummarizeCombinations, get_Iquartiles, get_Ispecies, get_ImRef
-from   abc_c6_aux           import make_plots, make_error_pdfs, TestStats, SummaryPDFs
+from   abc_c6_aux           import make_plots, make_plots_angstrom, TestStats, SummaryPDFs
 from   brdf                 import rtlsReflectance
 from   mcd43c               import BRDF
 from functools import reduce
@@ -88,7 +88,42 @@ def setupNN(self,retrieval,expid,
     self.nTarget = len(Target)
     self.K       = K
     self.nHidden = nHidden
-
+
+    # figure out if you need to calculate angstrom exponent
+    angstrom = False
+    for tname in Target:
+        if not angstrom:
+            if 'AE' in tname:
+                angstrom = True
+    self.angstrom = angstrom
+
+    # if angstrom is being trained
+    # find the base wavelength
+    # calculate angstrom with respect to the base wavelength
+    # -------------------------------------------------------
+    if angstrom:
+        # find base wavelength
+        for i,tname in enumerate(Target):
+            if 'Tau' in tname:
+                base_name = tname
+                base_wavs = tname.split('Tau')[-1]
+                base_wav = float(base_wavs)
+                base_tau = self.__dict__[tname]
+                base_wav_i = i
+
+        self.AE_base_wav = base_wav
+        self.AE_base_wav_i = base_wav_i
+
+        # Calculate the angstrom exponent
+        # with respect to the base wavelength
+        for tname in Target:
+            if 'Tau' not in tname:
+                wavs = tname.split('AE')[-1]
+                wav  = float(wavs)
+                tau  = self.__dict__['aTau'+wavs]
+                AE = -1.*np.log(tau/base_tau)/np.log(wav/base_wav)
+                self.__dict__['aAE'+wavs] = AE
+
     # Balance the dataset before splitting
     # No aerosol type should make up more that 35% 
     # of the total number of obs
@@ -151,20 +186,13 @@ class ABC(object):
     def __init__(self,fname,Albedo,coxmunk_lut=None,NDVI=False):
 
         # Get Auxiliary Data
-        self.setfnameRoot(fname)
+        self.fnameRoot = fname[:-3]
         self.setWind()
         self.setAlbedo(Albedo,coxmunk_lut=coxmunk_lut)
         self.setSpec()
         if NDVI:
             self.setNDVI()
 
-    def setfnameRoot(self,fname):
-        if self.sat == 'Aqua':
-            self.fnameRoot = 'myd_' + fname.split('/')[-1].split('.')[0]
-        elif self.sat == 'Terra':
-            self.fnameRoot = 'mod_' + fname.split('/')[-1].split('.')[0]
-
-
     def setWind(self):
         # Read in wind
         # ------------------------
@@ -1135,12 +1163,17 @@ def _test(mxd,expid,c,plotting=True):
         print('{} not found.  Need to train this combinatin of inputs'.format(netFile))
         raise
     else:
-      netFile = outdir+"/"+expid+'_Tau.net'
+      invars = mxd.comblist[0]
+      netFile = outdir+"/"+".".join(invars)+'_Tau.net'
 
     mxd.net = mxd.loadnet(netFile)
     mxd.Input = mxd.comblist[c]
     TestStats(mxd,mxd.K,c)
-    if plotting: make_plots(mxd,expid,ident,I=mxd.iTest)
+    if plotting: 
+        if mxd.angstrom:
+            make_plots_angstrom(mxd,expid,ident,I=mxd.iTest)
+        else:
+            make_plots(mxd,expid,ident,I=mxd.iTest)
   else:
     k = 1
     for iTrain, iTest in mxd.kf:
@@ -1166,20 +1199,26 @@ def _test(mxd,expid,c,plotting=True):
           print('{} not found.  Need to train this combinatin of inputs'.format(netFile))
           raise
       else:
-        netFile = outdir+"/"+expid+'.k={}_Tau.net'.format(str(k))
+        invars = mxd.comblist[0]
+        netFile = outdir+"/"+".".join(invars)+'.k={}_Tau.net'.format(str(k))
 
 
       mxd.net = mxd.loadnet(netFile)
       mxd.Input = mxd.comblist[c]      
       TestStats(mxd,k-1,c)
-      if plotting: make_plots(mxd,expid,'.k={}'.format(str(k)),I=mxd.iTest)
+      if plotting: 
+          if mxd.angstrom:
+              make_plots_angstrom(mxd,expid,'.k={}'.format(str(k)),I=mxd.iTest)
+          else:
+              make_plots(mxd,expid,'.k={}'.format(str(k)),I=mxd.iTest)
       k = k + 1    
 
 #---------------------------------------------------------------------
 def _trainMODIS(mxdx):
 
   if not mxdx.combinations:
-    _train(mxdx,mxdx.expid,0)
+    Input = mxdx.comblist[0]
+    _train(mxdx,'.'.join(Input),0)
   else:
     for c,Input in enumerate(mxdx.comblist):
       _train(mxdx,'.'.join(Input),c)