NASA Ocean Color

ocssw V2022
 '''
 * NAME: merge_afrt.py
 *
 * DESCRIPTION: Executable program to merge afrt (radiative
 * transfer) NetCDF4 files
  
 * USAGE: python merge_afrt.py -p [PCF file path] -wl [wavelength range] 
                 -sd [particle model range] -tau [optical depth range]
                 -wnd [wind speed range] 
  
 * Created on October 09, 2018
  
 * Author: Samuel Anderson
 '''
  
 import os
 import sys
 import subprocess
 from datetime import datetime as dt
 import optparse as optparse
 import logging
 LOG = logging.getLogger('afrt')
 import numpy as np
 import tables 
  
  
  
  
 def main():
  
     args = sys.argv
     
     description = '''Merge AFRT NetCDF output files.'''
     usage = "usage: python merge_afrt.py -d [dir] -wl [wavelength id range] -sd [model id range] -tau [tau id range] -wnd [wnd id range] -o [output directory] [options]"
     version = "v1"
  
     parser = optparse.OptionParser(description=description,usage=usage,version=version)
  
     # Mandatory arguments
     mandatoryGroup = optparse.OptionGroup(parser, "Mandatory Arguments",
                         "At a minimum these arguments must be specified")
  
     parser.add_option_group(mandatoryGroup)
  
     mandatoryGroup.add_option('-d','--idir',
                       action="store",
                       dest="idir" ,
                       type="string",
                       help="The input directory")
  
     # Optional arguments
     optionalGroup = optparse.OptionGroup(parser, "Extra Options",
                         "These options may be used to customize behavior of this program.")
  
     optionalGroup.add_option('-f','--fname',
                       action="store",
                       dest="fname" ,
                       type="string",
                       help="File name of intermediate product names")
  
     optionalGroup.add_option('-l','--wl',
                       action="store",
                       dest="wl" ,
                       type="int",
                       nargs=2,
                       help="Range of wavelength indices")
  
     optionalGroup.add_option('-s','--sd',
                       action="store",
                       dest="sd" ,
                       type="int",
                       nargs=2,
                       help="Range of particle model indices")
  
     optionalGroup.add_option('-t','--tau',
                       action="store",
                       dest="tau" ,
                       type="int",
                       nargs=2,
                       help="Range of optical depth indices")
  
     optionalGroup.add_option('-w','--wnd',
                       action="store",
                       dest="wnd" ,
                       type="int",
                       nargs=2,
                       help="Range of wind speed indices")
  
     optionalGroup.add_option('-o','--odir',
                       action="store",
                       dest="odir" ,
                       type="string",
                       help="The output directory")
  
     parser.add_option('-v', '--verbose',
                       dest='verbosity',
                       action="count",
                       default=0,
                       help='each occurrence increases verbosity 1 level from ERROR: -v=WARNING -vv=INFO -vvv=DEBUG')
  
     parser.add_option_group(optionalGroup)
  
     # Parse the arguments from the command line
     (opt, args) = parser.parse_args()
  
     # Set up the logging levels
     levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
     logging.basicConfig(level = levels[opt.verbosity])
  
     # Check that all of the mandatory options are given. If one or more 
     # are missing, print error message and exit...
     mandatories = []
     mand_errors = []
     isMissingMand = False
     for m,m_err in zip(mandatories,mand_errors):
         if not opt.__dict__[m]:
             isMissingMand = True
     if isMissingMand :
         parser.error("Incomplete mandatory arguments, aborting...")
         return
  
     # Check for mutually exclusive options
     LOG.info('Merging radiative transfer tables in  %s' % (opt.odir) )
     
     if os.path.isdir(opt.odir):
         o_path = opt.odir
     elif os.path.isdir(os.path.dirname(opt.odir)):
         o_path = opt.odir
         os.mkdir(o_path)            
     else:
         print ("Output path invalid")
         return
     if not os.path.isdir(opt.idir):
         os.mkdir(opt.idir)            
     os.chdir(opt.idir)
     if opt.fname:
         with open(opt.fname) as fp: 
             contents = fp.readlines()
             for fn in contents:
                 fn = fn.lstrip()
                 fn = fn.strip("\n")
                 command = "wget -N --content-disposition https://oceandata.sci.gsfc.nasa.gov/cgi/gethiddenfile/" + fn 
                 print (command)               
                 subprocess.run(command, shell=True)
     valid_files = []
     listing = os.listdir(opt.idir)
     listing.sort() 
     for x in listing:        
         if x[0] == ".":
             continue                
         fpath = opt.idir + "/" + x        
         if not os.path.isfile(fpath):
             continue            
         if not tables.is_hdf5_file(fpath):
             continue
         if not (x.find("RT") >= 0): 
             continue   
         try:
             h5file = tables.open_file(fpath, "r")
             toa = h5file.get_node('/', "TOA_RADIANCE")
             shp = toa.shape
             wl = h5file.get_node('/', "wavelength_index")
             if wl[0]<opt.wl[0] or wl[1]>opt.wl[1]:
                 h5file.close()
                 continue
             sd = h5file.get_node('/', "model_index")
             if sd[0]<opt.sd[0] or sd[1]>opt.sd[1]:
                 h5file.close()
                 continue
             tau = h5file.get_node('/', "tau_index")
             if tau[0]<opt.tau[0] or tau[1]>opt.tau[1]:
                 h5file.close()
                 continue
             wnd = h5file.get_node('/', "wind_index")
             if wnd[0]<opt.wnd[0] or wnd[1]>opt.wnd[1]:
                 h5file.close()
                 continue
             valid_files.append(fpath)
             h5file.close()
             stitle = x[0:3]
         except:
             h5file.close()
             continue     
     if len(valid_files) == 0:
         print( "Exiting...no valid files in input directory.")
         return
     
     sdd = opt.sd[1]-opt.sd[0]+1
     wld = opt.wl[1]-opt.wl[0]+1
     taud = opt.tau[1]-opt.tau[0]+1
     wndd = opt.wnd[1]-opt.wnd[0]+1
     szad = shp[4]
     thed = shp[5]
     phid = shp[6]
     stkd = shp[7]
  
     stitle = "RT00"
     rstr = "_MD%(s0)d-%(s1)d_WL%(l0)d-%(l1)d_TA%(t0)d-%(t1)d_WD%(w0)d-%(w1)d" % \
     {"s0":opt.sd[0],"s1":opt.sd[1],"l0":opt.wl[0],"l1":opt.wl[1],"t0":opt.tau[0],"t1":opt.tau[1],"w0":opt.wnd[0],"w1":opt.wnd[1]}
  
     ofilepath = opt.odir + "/" + stitle + rstr
     of = tables.open_file(ofilepath, "w", "Output of AF Radiative Transfer.")
     atom = tables.Float64Atom()
     BOA_RADIANCE_a = of.create_carray("/", 'BOA_RADIANCE', atom, (sdd,wld,taud,wndd,szad,thed,phid,stkd), "")
     BOA_DIFFUSE_a = of.create_carray("/", 'BOA_DIFFUSE', atom, (sdd,wld,taud,wndd,szad), "")
     BOA_DIRECT_a = of.create_carray("/", 'BOA_DIRECT', atom, (sdd,wld,taud,wndd,szad), "")
     HEM_REFLECTANCE_a = of.create_carray("/", 'HEM_REFLECTANCE', atom, (sdd,wld,taud,wndd,szad), "")
     SBAR_a = of.create_carray("/", 'SBAR', atom, (sdd,wld,taud,wndd,szad), "")
     TAU_AEROSOL_a = of.create_carray("/", 'TAU_AEROSOL', atom, (sdd,wld,taud,wndd), "")
     TAU_H2O_a = of.create_carray("/", 'TAU_H2O', atom, (sdd,wld,taud,wndd), "")
     TAU_OZONE_a = of.create_carray("/", 'TAU_OZONE', atom, (sdd,wld,taud,wndd), "")
     TAU_RAYLEIGH_a = of.create_carray("/", 'TAU_RAYLEIGH', atom, (sdd,wld,taud,wndd), "")
     TAU_TOTAL_a = of.create_carray("/", 'TAU_TOTAL', atom, (sdd,wld,taud,wndd), "")
     TOA_RADIANCE_a = of.create_carray("/", 'TOA_RADIANCE', atom, (sdd,wld,taud,wndd,szad,thed,phid,stkd), "")
     TOA_DIFFUSE_a = of.create_carray("/", 'TOA_DIFFUSE', atom, (sdd,wld,taud,wndd,szad), "")
 #    TRANSMITTANCE_a = of.create_carray("/", 'TRANSMITTANCE', atom, (sdd,wld,taud,wndd,szad), "")
     FTW_DIRECT_OCEAN_a = of.create_carray("/", 'FTW_DIRECT_OCEAN', atom, (sdd,wld,taud,wndd,szad), "")
     FTW_DIFFUSE_OCEAN_a = of.create_carray("/", 'FTW_DIFFUSE_OCEAN', atom, (sdd,wld,taud,wndd,szad), "")
     FTW_UP_DIFFUSE_OCEAN_BELOW_a = of.create_carray("/", 'FTW_UP_DIFFUSE_OCEAN_BELOW', atom, (sdd,wld,taud,wndd,szad), "")
     FTW_UP_DIFFUSE_OCEAN_ABOVE_a = of.create_carray("/", 'FTW_UP_DIFFUSE_OCEAN_ABOVE', atom, (sdd,wld,taud,wndd,szad), "")
     HEM_TRANS_REF_a = of.create_carray("/", 'HEM_TRANS_REF', atom, (sdd,wld,taud,wndd,szad), "")
     HEM_REFL_REF_a = of.create_carray("/", 'HEM_REFL_REF', atom, (sdd,wld,taud,wndd,szad), "")
     HEM_DIRECT_REF_a = of.create_carray("/", 'HEM_DIRECT_REF', atom, (sdd,wld,taud,wndd,szad), "")
     HEM_DIFFUSE_REF_a = of.create_carray("/", 'HEM_DIFFUSE_REF', atom, (sdd,wld,taud,wndd,szad), "")
     SZA_a = of.create_carray("/", 'SZA', atom, (szad,), "")
     PHI_a = of.create_carray("/", 'PHI', atom, (phid,), "")
     THETA_a = of.create_carray("/", 'THETA', atom, (thed,), "")
     AOT_550_a = of.create_carray("/", 'AOT_550', atom, (taud,), "")
     WAVELENGTH_a = of.create_carray("/", 'WAVELENGTH', atom, (wld,), "")
     WIND_SPEED_a = of.create_carray("/", 'WIND_SPEED', atom, (wndd,), "")
     atom = tables.Int32Atom()
     wavelength_index_a = of.create_carray("/", 'wavelength_index', atom, (2,), "")
     model_index_a = of.create_carray("/", 'model_index', atom, (2,), "")
     tau_index_a = of.create_carray("/", 'tau_index', atom, (2,), "")
     wind_index_a = of.create_carray("/", 'wind_index', atom, (2,), "")
     
     l = np.zeros((2,)) 
     s = np.zeros((2,)) 
     t = np.zeros((2,)) 
     w = np.zeros((2,)) 
     for fpath in valid_files: 
         try:       
             ifile = tables.open_file(fpath, "r")
             l[:] = ifile.get_node("/", "wavelength_index")[:]        
             s[:] = ifile.get_node("/", "model_index")[:]
             t[:] = ifile.get_node("/", "tau_index")[:]
             w[:] = ifile.get_node("/", "wind_index")[:]
             l[0]-=1
             s[0]-=1
             t[0]-=1
             w[0]-=1
             BOA_RADIANCE_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1],:,:,:,:] = \
             ifile.get_node("/", "BOA_RADIANCE")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:,:,:,:]
             BOA_DIFFUSE_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "BOA_DIFFUSE")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             BOA_DIRECT_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "BOA_DIRECT")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             HEM_REFLECTANCE_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "HEM_REFLECTANCE")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             SBAR_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1],:] = \
             ifile.get_node("/", "SBAR")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             TAU_AEROSOL_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]] = \
             ifile.get_node("/", "TAU_AEROSOL")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0]]
             TAU_H2O_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]] = \
             ifile.get_node("/", "TAU_H2O")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0]]
             TAU_OZONE_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]] = \
             ifile.get_node("/", "TAU_OZONE")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0]]
             TAU_RAYLEIGH_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]] = \
             ifile.get_node("/", "TAU_RAYLEIGH")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0]]
             TAU_TOTAL_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]] = \
             ifile.get_node("/", "TAU_TOTAL")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0]]
             TOA_RADIANCE_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1],:,:,:,:] = \
             ifile.get_node("/", "TOA_RADIANCE")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:,:,:,:]
             TOA_DIFFUSE_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1],:] = \
             ifile.get_node("/", "TOA_DIFFUSE")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
 #            TRANSMITTANCE_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1],:] = \
 #            ifile.get_node("/", "TRANSMITTANCE")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             FTW_DIRECT_OCEAN_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "FTW_DIRECT_OCEAN")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             FTW_DIFFUSE_OCEAN_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "FTW_DIFFUSE_OCEAN")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             FTW_UP_DIFFUSE_OCEAN_BELOW_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "FTW_UP_DIFFUSE_OCEAN_BELOW")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             FTW_UP_DIFFUSE_OCEAN_ABOVE_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "FTW_UP_DIFFUSE_OCEAN_ABOVE")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             HEM_TRANS_REF_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "HEM_TRANS_REF")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             HEM_REFL_REF_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "HEM_REFL_REF")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             HEM_DIRECT_REF_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "HEM_DIRECT_REF")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             HEM_DIFFUSE_REF_a[s[0]:s[1],l[0]:l[1],t[0]:t[1],w[0]:w[1]:] = \
             ifile.get_node("/", "HEM_DIFFUSE_REF")[0:s[1]-s[0],0:l[1]-l[0],0:t[1]-t[0],0:w[1]-w[0],:]
             SZA_a[:] = ifile.get_node("/", "SZA")[:]
             PHI_a[:] = ifile.get_node("/", "PHI")[:]
             THETA_a[:] = ifile.get_node("/", "THETA")[:]
             AOT_550_a[t[0]:t[1]] = ifile.get_node("/", "AOT")[:]
             WAVELENGTH_a[l[0]:l[1]] = ifile.get_node("/", "WAVELENGTH")[:]
             WIND_SPEED_a[w[0]:w[1]] = ifile.get_node("/", "WIND_SPEED")[:]
             ifile.close()
             of.flush()
         except:
             ifile.close()
             continue
  
     wavelength_index_a[:] = opt.wl[:]
     model_index_a[:] = opt.sd[:]
     tau_index_a[:] = opt.tau[:]
     wind_index_a[:] = opt.wnd[:]
     of.flush()
     of.close()
  
     print("Merge complete at " + ofilepath)
     LOG.info('Exiting...')
     return 0
  
 if __name__=='__main__':
     sys.exit(main())        
         
merge_afrt.main
def main()
Main Function #.
Definition: merge_afrt.py:31