NASA Logo
Ocean Color Science Software

ocssw V2022
get_obpg_file_type.py
Go to the documentation of this file.
1 
2 """
3 A class for determining the OBPG type of a file.
4 """
5 __version__ = '2.0'
6 
7 __author__ = 'melliott'
8 
9 import calendar
10 import datetime
11 import math
12 import optparse
13 import os
14 import re
15 import subprocess
16 import sys
17 import tarfile
18 import time
21 
22 
23 KNOWN_SENSORS = ['Aquarius', 'CZCS', 'HAWKEYE','HICO',
24  'HMODISA', 'HMODIST', 'MERIS', 'MODISA',
25  'MODIS Aqua', 'MODIST', 'MODIS Terra',
26  'MOS', 'OCI', 'OCIS', 'OCM2', 'OCTS', 'OSMI',
27  'SeaWiFS', 'SGLI','VIIRSN', 'VIIRSJ1', 'VIIRSJ2']
28 
29 MONTH_ABBRS = dict((v.upper(), k) for k, v in enumerate(calendar.month_abbr))
30 
31 
33  """
34  Converts a number of milliseconds to a string representing the time of day.
35  """
36  secs = math.trunc(millisecs / 1000)
37  hrs = math.trunc(secs / 3600)
38  secs = secs - (hrs * 3600)
39  mins = math.trunc(secs / 60)
40  secs = secs - (mins * 60)
41  tm_str = '{0:02}{1:02}{2:02}'.format(hrs, mins, secs)
42  return tm_str
43 
44 def get_timestamp_from_month_day(sdate, stime):
45  """
46  Creates timestamp for VIIRS L1.
47  """
48  year = int(sdate[0:4])
49  mon = int(sdate[4:6])
50  dom = int(sdate[6:8])
51  hrs = int(stime[0:2])
52  mins = int(stime[2:4])
53  secs = int(stime[4:6])
54  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
55  return dt_obj.strftime('%Y%j%H%M%S')
56 
57 def get_timestamp_from_year_day_mil(year, doy, millisecs):
58  """
59  Return a timestamp in the YYYYDDDHHMMSS form from a given year (yr), day
60  of year (doy) and milliseconds (after 00:00 - millisecs).
61  """
62  time_str = convert_millisecs_to_time_str(millisecs)
63  timestamp = '{0:04}{1:03}{2}'.format(year, doy, time_str)
64  return timestamp
65 
66 class ObpgFileTyper(object):
67  """
68  A class containing methods for finding the type of an OBPG file (e.g.
69  MODIS L2b, SeaWiFS L1A, Aquarius L3m, etc.
70  """
71 
72  def __init__(self, fpath):
73  """
74  Save the path of the file in question and set up default
75  values for several thing still to be found.
76  """
77  if os.path.exists(fpath):
78  if tarfile.is_tarfile(fpath):
79  # self.file_path = self._extract_viirs_sdr(fpath)
80  self._extract_viirs_sdr(fpath)
81  if not self.file_path:
82  err_msg = "Error! Cannot process file {0}.".format(fpath)
83  sys.exit(err_msg)
84  else:
85  self.file_path = fpath
86  self.file_type = 'unknown'
87  self.instrument = 'unknown'
88  self.start_time = 'unknown'
89  self.end_time = 'unknown'
90  self.attributes = None
91  self.l0_data = None
92  else:
93  err_msg = "Error! File {0} could not be found.".format(fpath)
94  sys.exit(err_msg)
95 
96 # def _convert_month_day_to_doy(self, sdate, stime):
97 # """
98 # Creates timestamp for VIIRS L1.
99 # """
100 # year = int(sdate[0:4])
101 # mon = int(sdate[4:6])
102 # dom = int(sdate[6:8])
103 # hrs = int(stime[0:2])
104 # mins = int(stime[2:4])
105 # secs = int(stime[4:6])
106 # dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
107 # return dt_obj.strftime('%Y%j%H%M%S')
108 
109  def _create_modis_l1_timestamp(self, rng_date, rng_time):
110  """
111  Returns a date/time stamp for a MODIS L1 file from the appropriate
112  RANGINGDATE and RANGINGTIME attributes. The returned date/time stamp
113  is of form YYYYDDDHHMMSS, where YYYY = year, DDD = day of year,
114  HH = hour, MM = minute, and SS = second.
115  """
116  year = int(rng_date[0:4])
117  mon = int(rng_date[5:7])
118  dom = int(rng_date[8:10])
119  hrs = int(rng_time[0:2])
120  mins = int(rng_time[3:5])
121  secs = int(rng_time[6:8])
122  milsecs = int(rng_time[9:15])
123  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs, milsecs)
124  if milsecs >= 500000:
125  dt_delta = datetime.timedelta(seconds=1)
126  dt_obj += dt_delta
127  return dt_obj.strftime('%Y%j%H%M%S')
128 
129  def _create_octs_l1_timestamp(self, dt_str):
130  """
131  Creates a timestamp for an OCTS L1.
132  """
133  year = int(dt_str[0:4])
134  mon = int(dt_str[4:6])
135  dom = int(dt_str[6:8])
136  hrs = int(dt_str[9:11])
137  mins = int(dt_str[12:14])
138  secs = int(dt_str[15:17])
139  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
140  return dt_obj.strftime('%Y%j%H%M%S')
141 
142  def _create_timestamp_using_mon_abbr(self, time_str):
143  """
144  Returns a properly formatted date/time stamp for MERIS L1B files from
145  an attribute in the file.
146  """
147  # Todo: Check that MERIS' and Python's month abbreviations match up ...
148  year = int(time_str[7:11])
149  mon = int(MONTH_ABBRS[time_str[3:6]])
150  dom = int(time_str[0:2])
151  hrs = int(time_str[12:14])
152  mins = int(time_str[15:17])
153  secs = int(time_str[18:20])
154  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
155  return dt_obj.strftime('%Y%j%H%M%S')
156 
157  def _create_timestamp_using_YMDTHMS(self, time_str):
158  """
159  Returns a properly formatted date/time stamp for MERIS L1B files from
160  an attribute in the file.
161  """
162  # Todo: Check that MERIS' and Python's month abbreviations match up ...
163  year = int(time_str[0:4])
164  mon = int(time_str[5:7])
165  dom = int(time_str[8:10])
166  hrs = int(time_str[11:13])
167  mins = int(time_str[14:16])
168  secs = int(time_str[17:19])
169  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
170  return dt_obj.strftime('%Y%j%H%M%S')
171 
172  def _extract_viirs_sdr(self, tar_path):
173  self.file_path = None
174  try:
175  # orig_path = os.getcwd()
176  tar_file = tarfile.TarFile(tar_path)
177  tar_members = tar_file.getnames()
178  for mbr in tar_members:
179  if mbr.startswith('SVM01'):
180  mbr_info = tar_file.getmember(mbr)
181  tar_file.extractall(members=[mbr_info], path='/tmp')
182  self.file_path = os.path.join('/tmp', mbr)
183  break
184  tar_file.close()
185  except:
186  exc_info = sys.exc_info()
187  for item in exc_info:
188  print(str(item))
189 
190  def get_attributes(self):
191  return self.attributes
192 
193  def _get_data_from_anc_attributes(self):
194  """
195  Processes Ancillary data files.
196  """
197  instrument = 'Ancillary'
198  file_type = self.attributes['Data Type']
199  return file_type, instrument
200 
201  def _get_data_from_l0_attributes(self):
202  """
203  Get the instrument and file type from the attributes for an L0 file.
204  """
205  file_type = 'unknown'
206  # instrument = 'unknown'
207  title_parts = self.attributes['Title'].split()
208  if title_parts[1].find('Level-0') != -1:
209  file_type = 'Level 0'
210  instrument = title_parts[0].strip()
211  return file_type, instrument
212 
213  def _get_data_from_l1_attributes(self, title):
214  """
215  Get the instrument and file type from the attributes for an L1 file.
216  """
217  file_type = 'Level 1'
218  instrument = 'unknown'
219  possible_levels = {'Level 1A': 'Level 1A', 'Level-1A': 'Level 1A',
220  'L1A': 'Level 1A', 'L1a': 'Level 1A',
221  'Level 1B': 'Level 1B', 'Level-1B': 'Level 1B',
222  'L1B': 'Level 1B', 'L1b': 'Level 1B'}
223 # ks = known_sensors
224  if title.find('Level 1') != -1:
225  working_title = title.replace('Level 1', 'Level-1')
226  else:
227  working_title = title
228  working_title = re.sub("""^['"](.*)['"].*;$""", '\g<1>', working_title)
229  title_parts = working_title.split()
230  for part in title_parts:
231  if part in KNOWN_SENSORS:
232  instrument = part
233  else:
234  # found = False
235  for ks in KNOWN_SENSORS:
236  if part.find(ks) != -1:
237  instrument = ks
238  # found = True
239  break
240  if part in possible_levels:
241  file_type = possible_levels[part]
242  if title.find('Browse Data') != -1:
243  file_type += ' Browse Data'
244  return file_type, instrument
245 
246  def _get_data_from_l2_attributes(self):
247  """
248  Get the instrument and file type from the attributes for an 20 file.
249  """
250  file_type = 'Level 2'
251  if 'Title' in self.attributes:
252  title_parts = self.attributes['Title'].split()
253  if self.attributes['Title'].find('Browse Data') != -1:
254  file_type += ' Browse Data'
255  elif 'title' in self.attributes:
256  title_parts = self.attributes['title'].split()
257  if self.attributes['title'].find('Browse Data') != -1:
258  file_type += ' Browse Data'
259  instrument = title_parts[0].strip()
260  return file_type, instrument
261 
262  def _get_data_from_l3_attributes(self):
263  """
264  Get the instrument and file type from the attributes for an L3 file.
265  """
266  file_type = 'unknown'
267  if 'Title' in self.attributes:
268  working_title = self.attributes['Title']
269  elif 'title' in self.attributes:
270  working_title = self.attributes['title']
271  elif 'instrument' in self.attributes:
272  pass
273  if (working_title.find('Level-3 Binned') != -1) or \
274  (working_title.find('level-3_binned') != -1):
275  file_type = 'Level 3 Binned'
276  elif working_title.find('Level-3 Standard Mapped Image') != -1:
277  file_type = 'Level 3 SMI'
278  instrument = self._get_instrument()
279  return file_type, instrument
280 
281  def _get_instrument(self):
282  """
283  Get the instrument from the attributes.
284  """
285  instrument = 'unknown'
286  if 'Title' in self.attributes or 'title' in self.attributes:
287  if 'Title' in self.attributes:
288  title_parts = self.attributes['Title'].split()
289  else:
290  title_parts = self.attributes['title'].split()
291  if title_parts[0].find('Level') == -1:
292  instrument = title_parts[0].strip()
293  elif 'instrument' in self.attributes:
294  instrument = self.attributes['instrument']
295  if not (instrument in KNOWN_SENSORS):
296  if instrument != 'unknown':
297  if 'platform' in self.attributes:
298  instrument = ' '.join([instrument,
299  self.attributes['platform']])
300  else:
301  if 'Sensor Name' in self.attributes:
302  instrument = self.attributes['Sensor Name'].strip()
303  if ('Sensor' in self.attributes) and not \
304  (instrument in KNOWN_SENSORS):
305  instrument = self.attributes['Sensor'].strip()
306  return instrument
307 
308  def _get_l0_constructor_data(self):
309  """
310  Try to get data from the MODIS L0 constructor for this file.
311  """
312  l0cnst_results = None
313  cmd = ['l0cnst_write_modis', '-n', self.file_path]
314  try:
315  l0_out = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE,
316  stderr=subprocess.PIPE).communicate()[0]
317  out_lines = l0_out.decode('utf-8').split('\n')
318  is_l0 = False
319  for line in out_lines:
320  if line.find('Total packets') != -1:
321  parts = line.split()
322  packets_read = float(parts[0])
323  if packets_read > 0.0:
324  is_l0 = True
325  break
326  if is_l0:
327  starttime, stoptime = self._get_l0_start_stop_times(out_lines)
328  if starttime and stoptime:
329  l0cnst_results = {'starttime' : starttime,
330  'stoptime' : stoptime}
331  except Exception:
332  l0cnst_results = None
333  return l0cnst_results
334 
335  def _get_l0_times(self):
336  """
337  Returns the start and end times for L0 data files.
338  """
339  if isinstance(self.l0_data, dict):
340  start_time = self.l0_data['starttime']
341  end_time = self.l0_data['stoptime']
342  else:
343  for line in self.l0_data:
344  if line.find('starttime') != -1:
345  time_part = line.split('=')[1]
346  start_time = time_part[0:4] + time_part[5:7] + \
347  time_part[8:10] + time_part[11:13] + \
348  time_part[14:16] + time_part[17:19]
349  if line.find('stoptime') != -1:
350  time_part = line.split('=')[1]
351  end_time = time_part[0:4] + time_part[5:7] + \
352  time_part[8:10] + time_part[11:13] +\
353  time_part[14:16] + time_part[17:19]
354  return start_time, end_time
355 
356  def _get_landsat_end_time(self, start_yr, start_doy, start_hr, start_min,
357  start_sec):
358  """
359  Compute and return the end time for a Landsat OLI data file, given the
360  start year, day of year, hour, mine, and second.
361  """
362  end_yr = start_yr
363  end_doy = start_doy
364  end_hr = int(start_hr)
365  end_min = int(start_min)
366  end_sec = int(start_sec) + 1
367  if end_sec >= 60:
368  end_sec = 0
369  end_min += 1
370  if end_min >= 60:
371  end_min = 0
372  end_hr += 1
373  if end_hr >= 24:
374  end_hr = 0
375  end_doy += 1
376  if end_yr % 4 == 0:
377  if end_doy > 366:
378  end_doy = 1
379  end_yr += 1
380  elif end_doy > 365:
381  end_doy = 1
382  end_yr += 1
383  end_time = '{0:4d}{1:03d}{2:02d}{3:02d}{4:02d}'.format(end_yr,
384  end_doy,
385  end_hr,
386  end_min,
387  end_sec)
388  return end_time
389 
390  def _get_landsat_times(self):
391  """
392  Computes and returns the start and end times for a Landsat OLI
393  data file.
394  """
395  # The Landsat OLI metadata only contains an acquisition date and
396  # scene center time. (For now) It is assumed that the scene center
397  # time is the start time and the end time is one second later.
398  acquisition_date = self.attributes['DATE_ACQUIRED'].strip('"')
399  center_time = self.attributes['SCENE_CENTER_TIME'].strip('"')
400 
401  start_yr = int(acquisition_date[0:4])
402  start_mon = int(acquisition_date[5:7])
403  start_dom = int(acquisition_date[8:10])
404  start_doy =seadasutils.time_utils.convert_month_day_to_doy(start_mon, start_dom,
405  start_yr)
406  start_date_str = '{0:04d}{1:03d}'.format(start_yr, start_doy)
407  start_hr = center_time[0:2]
408  start_min = center_time[3:5]
409  start_sec = center_time[6:8]
410  start_time = ''.join([start_date_str, start_hr, start_min,
411  start_sec])
412  end_time = self._get_landsat_end_time(start_yr, start_doy,
413  start_hr, start_min, start_sec)
414  return start_time, end_time
415 
416  def _get_time_from_coverage_field(self, coverage_time):
417  """
418  Returns the stamp computed from the coverage_time
419  """
420  if coverage_time[0:4] == '':
421  yr = '0000'
422  else:
423  yr = '{0:04d}'.format(int(coverage_time[0:4]))
424  doy = '{0:03d}'.format(
426  coverage_time[8:10],
427  coverage_time[0:4]))
428  if coverage_time[11:13] == '':
429  hr = '00'
430  else:
431  hr = '{0:02d}'.format(int(coverage_time[11:13]))
432  if coverage_time[14:16] == '':
433  min = '00'
434  else:
435  min = '{0:02d}'.format(int(coverage_time[14:16]))
436  if coverage_time[17:19] == '':
437  sec = '00'
438  else:
439  sec = '{0:02d}'.format(int(coverage_time[17:19]))
440  time_stamp = ''.join([yr, str(doy), hr, min, sec])
441  return time_stamp
442 
443 
445  """
446  API method to return the file's attributes.
447  """
448  return self.attributes
449 
450  def get_file_times(self):
451  """
452  Returns the start and end time for a file.
453  """
454 # print 'self.instrument: "' + self.instrument + '"'
455  start_time = 'unable to determine start time'
456  end_time = 'unable to determine end time'
457  if 'VIIRS' in self.instrument and self.file_type == 'SDR':
458  start_time = get_timestamp_from_month_day(
459  self.attributes['Beginning_Date'],
460  self.attributes['Beginning_Time'])
461  end_time = get_timestamp_from_month_day(
462  self.attributes['Ending_Date'],
463  self.attributes['Ending_Time'])
464  elif self.file_type.find('Level 1') != -1:
465  start_time, end_time = self._get_l1_times()
466  elif self.file_type.find('Level 2') != -1 or \
467  self.file_type.find('Level 3') != -1:
468  try:
469  if 'time_coverage_start' in self.attributes:
470  start_time = self._get_time_from_coverage_field(self.attributes['time_coverage_start'])
471  end_time = self._get_time_from_coverage_field(self.attributes['time_coverage_end'])
472  elif 'RANGEBEGINNINGDATE' in self.attributes:
473  start_time, end_time = self._get_l1_modis_times()
474  elif 'Start Time' in self.attributes:
475  start_time = self.attributes['Start Time'][0:13]
476  elif 'time_coverage_start' in self.attributes:
477  start_time = self._get_time_from_coverage_field(
478  self.attributes['time_coverage_start'])
479  elif 'Start Day' in self.attributes:
480  start_time = get_timestamp_from_year_day_mil(
481  int(self.attributes['Start Year']),
482  int(self.attributes['Start Day']),
483  int(self.attributes['Start Millisec'])
484  )
485  if 'End Time' in self.attributes:
486  end_time = self.attributes['End Time'][0:13]
487  elif 'time_coverage_end' in self.attributes:
488  end_time = self._get_time_from_coverage_field(
489  self.attributes['time_coverage_end'])
490  elif 'End Day' in self.attributes:
492  int(self.attributes['End Year']),
493  int(self.attributes['End Day']),
494  int(self.attributes['End Millisec'])
495  )
496  except KeyError:
497  if self.instrument.find('Aquarius') != -1:
498  stime_str = convert_millisecs_to_time_str(
499  int(self.attributes['Start Millisec']))
500  start_time = '{0:04}{1:03}{2}'.format(
501  int(self.attributes['Start Year']),
502  int(self.attributes['Start Day']),
503  stime_str)
504  etime_str = convert_millisecs_to_time_str(
505  int(self.attributes['Start Millisec']))
506  end_time = '{0:04}{1:03}{2}'.format(
507  int(self.attributes['End Year']),
508  int(self.attributes['End Day']),
509  etime_str)
510  else:
511  raise
512  elif self.file_type.find('Level 0') != -1:
513  start_time, end_time = self._get_l0_times()
514  return start_time, end_time
515 
516  def _get_type_using_platform(self):
517  levelMap = {'L0':'Level 0',
518  'L1A':'Level 1A',
519  'L1C':'Level 1C',
520  'GEO':'GEO',
521  'L1B':'Level 1B',
522  'L2':'Level 2',
523  'L3 Binned':'Level 3 Binned',
524  'L3 Mapped':'Level 3 SMI'
525  }
526  instrumentMap = {'SEAWIFS': "SeaWiFS",
527  "MOS": "MOS",
528  "OCTS": "OCTS",
529  "AVHRR":"AVHRR",
530  "OSMI": "OSMI",
531  "CZCS": "CZCS",
532  "OCM1": "OCM1",
533  "OCM2": "OCM2",
534  "OCM-2": "OCM2",
535  "MERIS":"MERIS",
536  "OCRVC":"OCRVC",
537  "HICO": "HICO",
538  "GOCI": "GOCI",
539  # "OLI": "OLI",
540  "AQUARIUS": "Aquarius" ,
541  "OCIA": "OCIA",
542  "AVIRIS": "AVIRIS",
543  "PRISM": "PRISM",
544  "SGLI": "SGLI",
545  "L5TM": "L5TM",
546  "L7ETM": "L7ETM",
547  "HAWKEYE": "HAWKEYE",
548  "MISR": "MISR",
549  "OCI": "OCI"
550  }
551 
552  if self.attributes['processing_level'] in levelMap:
553  self.file_type = levelMap[self.attributes['processing_level']]
554  if 'title' in self.attributes:
555  if self.attributes['title'].find('Geolocation') != -1:
556  self.file_type = 'GEO'
557  if self.file_type == 'Level 3 SMI':
558  if self.attributes['title'].find('Standard Mapped Image') == -1:
559  self.file_type == 'Level 3 Mapped'
560  inst = self.attributes['instrument'].upper()
561  if inst in instrumentMap:
562  self.instrument = instrumentMap[inst]
563  return True
564  plat = self.attributes['platform'].upper()
565  if inst.find('MODIS') != -1:
566  if plat.find('AQUA') != -1:
567  self.instrument = "MODIS Aqua"
568  return True
569  elif plat.find('TERRA') != -1:
570  self.instrument = "MODIS Terra"
571  return True
572  elif inst.find('VIIRS') != -1:
573  if plat.find('SUOMI-NPP') != -1:
574  self.instrument = "VIIRS NPP"
575  return True
576  elif plat.find('JPSS-1') != -1:
577  self.instrument = "VIIRS J1"
578  return True
579  elif plat.find('JPSS-2') != -1:
580  self.instrument = "VIIRS J2"
581  return True
582  elif inst.find('OLCI') != -1:
583  if plat == 'SENTINEL-3' or plat.find('3A') != -1:
584  self.instrument = "OLCI S3A"
585  return True
586  elif plat.find('3B') != -1:
587  self.instrument = "OLCI S3B"
588  return True
589  elif inst.find('MSI') != -1:
590  if plat == 'SENTINEL-2' or plat.find('2A') != -1:
591  self.instrument = "MSI S2A"
592  return True
593  elif plat.find('2B') != -1:
594  self.instrument = "MSI S2B"
595  return True
596  elif inst.find('OLI') != -1:
597  if plat == 'LANDSAT-8':
598  self.instrument = "OLI L8"
599  return True
600  elif plat == 'LANDSAT-9':
601  self.instrument = "OLI L9"
602  return True
603  return False
604 
605  def _get_file_type_from_attributes(self):
606  """
607  Determines the file type and instrument from the file attributes and
608  sets those values in the object.
609  """
610 
611  if ('platform' in self.attributes) and ('instrument' in self.attributes) and ('processing_level' in self.attributes):
612  if self._get_type_using_platform():
613  return
614 
615  if 'Title' in self.attributes or 'title' in self.attributes:
616  self._get_type_using_title()
617  elif 'ASSOCIATEDINSTRUMENTSHORTNAME' in self.attributes:
619  elif 'PRODUCT' in self.attributes:
620  if self.attributes['PRODUCT'].find('MER_') != -1:
621  self.instrument = 'MERIS'
622  self.file_type = 'Level 1B'
623  elif 'Instrument_Short_Name' in self.attributes:
624  self.instrument = self.attributes['Instrument_Short_Name'].upper()
625  if self.instrument == 'HICO':
626  if self.attributes['processing_level'].find('L1B') != -1:
627  self.file_type = 'Level 1B'
628  if self.instrument == 'VIIRS':
629  if 'Mission_Name' in self.attributes:
630  mission = self.attributes['Mission_Name']
631  if mission == 'NPP':
632  self.instrument = 'VIIRS NPP'
633  elif mission == 'JPSS-1':
634  self.instrument = 'VIIRS J1'
635  if 'N_Dataset_Type_Tag' in self.attributes:
636  self.file_type = self.attributes['N_Dataset_Type_Tag']
637  elif 'Product Level' in self.attributes:
638  if 'Sensor name' in self.attributes:
639  self.instrument = self.attributes['Sensor name']
640  self.file_type = self.attributes['Product Level']
641  if not self.file_type.startswith('Level'):
642  self.file_type = 'Level ' + self.file_type
643  elif 'Sensor' in self.attributes:
644  if self.attributes['Sensor'].find('SGLI') != -1:
645  self.instrument = 'SGLI'
646  self.file_type = 'Level 1B'
647  elif 'SENSOR_ID' in self.attributes:
649 
650  def get_file_type(self):
651  """
652  Returns what type (L1A, L2, etc.) a file is and what
653  platform/sensor/instrument made the observations.
654  """
655  orig_path = None
656  self._read_metadata()
657  if self.attributes:
659  else:
661  if self.instrument.find('MODISA') != -1:
662  self.instrument = 'MODIS Aqua'
663  if self.instrument.find('MODIST') != -1:
664  self.instrument = 'MODIS Terra'
665  if orig_path:
666  self.file_path = orig_path
667  return self.file_type, self.instrument
668 
669  def _get_file_type_landsat(self):
670  """
671  Sets the file type and instrument for Landsat OLI data files
672  """
673  # Landsat 8 OLI
674  if self.attributes['SENSOR_ID'] == '"OLI"' or self.attributes['SENSOR_ID'] == '"OLI_TIRS"':
675  if self.attributes['SPACECRAFT_ID'].find('LANDSAT_8') != -1:
676  self.instrument = 'OLI L8'
677  else:
678  self.instrument = 'OLI L9'
679  else:
680  self.instrument = self.attributes['SENSOR_ID'].\
681  strip().strip('"')
682  if 'DATA_TYPE' in self.attributes:
683  if self.attributes['DATA_TYPE'].find('L1G') != -1 or \
684  self.attributes['DATA_TYPE'].find('L1GT') != -1 or \
685  self.attributes['DATA_TYPE'].find('L1P') != -1 or \
686  self.attributes['DATA_TYPE'].find('L1T') != -1:
687  self.file_type = 'Level 1B'
688  elif 'PROCESSING_LEVEL' in self.attributes:
689  if self.attributes['PROCESSING_LEVEL'].find('L1G') != -1 or \
690  self.attributes['PROCESSING_LEVEL'].find('L1GT') != -1 or \
691  self.attributes['PROCESSING_LEVEL'].find('L1P') != -1 or \
692  self.attributes['PROCESSING_LEVEL'].find('L1T') != -1 or\
693  self.attributes['PROCESSING_LEVEL'].find('L1TP') != -1:
694  self.file_type = 'Level 1B'
695  else:
696  self.file_type = 'Level ' + self.attributes['DATA_TYPE'].\
697  strip().strip('"')
698  def get_sensor(self):
699  """
700  Returns what type (L1A, L2, etc.) a file is and what
701  platform/sensor/instrument made the observations.
702  """
703  orig_path = None
704  self._read_metadata()
705  if self.attributes:
707  else:
709  sensor = self.instrument
710  if re.search ('OLI', self.instrument):
711  sensor = 'OLI'
712  # if re.search ('L8', self.instrument):
713  # sensor = 'OLIL8'
714  # elif re.search ('L9', self.instrument):
715  # sensor = 'OLIL9'
716  elif re.search('Aqua', self.instrument):
717  sensor = 'MODISA'
718  elif re.search('Terra', self.instrument,):
719  sensor = 'MODIST'
720  elif re.search('OLCI S3A', self.instrument):
721  sensor = 'OLCIS3A'
722  elif re.search('OLCI S3B', self.instrument,):
723  sensor = 'OLCIS3B'
724  elif re.search('MSI S2A', self.instrument,):
725  sensor = 'MSIS2A'
726  elif re.search('MSI S2B', self.instrument,):
727  sensor = 'MSIS2B'
728  elif re.search('NPP', self.instrument,):
729  sensor = 'VIIRSN'
730  elif re.search('J1', self.instrument,):
731  sensor = 'VIIRSJ1'
732  elif re.search('J2', self.instrument,):
733  sensor = 'VIIRSJ2'
734  if orig_path:
735  self.file_path = orig_path
736  return sensor
737 
738  def _get_l0_start_stop_times(self, l0_lines):
739  """
740  Returns the start time and stop time if found in l0_lines.
741  """
742  starttime = None
743  stoptime = None
744  for line in l0_lines:
745  if line.find('starttime') != -1:
746  starttime = line.strip().split('=')[1]
747  if line.find('stoptime') != -1:
748  stoptime = line.strip().split('=')[1]
749  return starttime, stoptime
750 
751  def _get_l1_goci_times(self):
752  """
753  Finds and returns timestamps for GOCI L1 data files.
754  """
755  if 'Scene Start time' in self.attributes and self.attributes['Scene Start time'] != 'unknown':
756  # start_time = self.attributes['Scene Start time']
757  start_time = self._create_timestamp_using_mon_abbr(self.attributes['Scene Start time'])
758  elif 'start_time' in self.attributes and self.attributes['start_time'] != 'unknown':
759  start_time = self.attributes['start_time']
760  elif 'processing start time' in self.attributes:
761  st_yr = int(self.attributes['processing start time'][7:11])
762  mon = time.strptime(self.attributes['processing start time']\
763  [3:6], '%b').tm_mon
764  dom = int(self.attributes['processing start time'][0:2])
766  start_time = '{0:04d}{1:03d}{2:02d}{3}{4}'.format(st_yr, doy,
767  int(self.attributes['processing start time'][12:14]),
768  self.attributes['processing start time'][15:17],
769  self.attributes['processing start time'][18:20])
770  if 'Scene end time' in self.attributes and self.attributes['Scene end time'] != 'unknown':
771  end_time = self._create_timestamp_using_mon_abbr(self.attributes['Scene end time'])
772  elif 'end_time' in self.attributes and self.attributes['end_time'] != 'unknown':
773  end_time = self.attributes['end_time']
774  elif 'processing end time' in self.attributes:
775  end_yr = int(self.attributes['processing end time'][7:11])
776  mon = time.strptime(self.attributes['processing end time']\
777  [3:6], '%b').tm_mon
778  dom = int(self.attributes['processing end time'][0:2])
780  end_time = '{0:04d}{1:03d}{2:02d}{3}{4}'.format(end_yr, doy,
781  int(self.attributes['processing end time'][12:14]),
782  self.attributes['processing end time'][15:17],
783  self.attributes['processing end time'][18:20])
784  return start_time, end_time
785 
786  def _get_l1_hico_times(self):
787  """
788  Finds and returns timestamps for HICO L1 data files.
789  """
790  start_time = get_timestamp_from_month_day(\
791  self.attributes['Beginning_Date'],
792  self.attributes['Beginning_Time'])
793  end_time = get_timestamp_from_month_day(
794  self.attributes['Ending_Date'],
795  self.attributes['Ending_Time'])
796  return start_time, end_time
797 
798  def _get_l1_meris_times(self):
799  """
800  Finds and returns timestamps for MERIS L1 data files.
801  """
802  if 'FIRST_LINE_TIME' in self.attributes:
803  start_time = self._create_timestamp_using_mon_abbr(
804  self.attributes['FIRST_LINE_TIME'])
805  end_time = self._create_timestamp_using_mon_abbr(
806  self.attributes['LAST_LINE_TIME'])
807  elif 'startTime' in self.attributes:
808  start_time = self._create_timestamp_using_YMDTHMS(
809  self.attributes['startTime'])
810  end_time = self._create_timestamp_using_YMDTHMS(
811  self.attributes['stopTime'])
812  else:
813  start_time = self._create_timestamp_using_mon_abbr(
814  self.attributes['start_date'].strip('"'))
815  end_time = self._create_timestamp_using_mon_abbr(
816  self.attributes['stop_date'].strip('"'))
817  return start_time, end_time
818 
819  def _get_l1_modis_times(self):
820  """
821  Finds and returns timestamps for MODIS L1 data files.
822  """
823  start_time = self._create_modis_l1_timestamp(
824  self.attributes['RANGEBEGINNINGDATE'],
825  self.attributes['RANGEBEGINNINGTIME'])
826  end_time = self._create_modis_l1_timestamp(
827  self.attributes['RANGEENDINGDATE'],
828  self.attributes['RANGEENDINGTIME'])
829  return start_time, end_time
830 
831  def _get_l1_msi_times(self):
832  """
833  Finds and returns timestamps for MSI L1 data files.
834  """
835  if 'startTime' in self.attributes:
836  start_time = self._create_timestamp_using_YMDTHMS(
837  self.attributes['startTime'])
838  end_time = start_time
839  return start_time, end_time
840 
841  def _get_l1_ocm2_times(self):
842  """
843  Finds and returns timestamps for OCM2 L1 data files.
844  """
845  #yr, doy, millisecs
846  if 'Start Year' in self.attributes:
847  start_time = get_timestamp_from_year_day_mil(\
848  self.attributes['Start Year'],
849  self.attributes['Start Day'],
850  self.attributes['Start Millisec'])
851  elif 'time_coverage_start' in self.attributes:
852  start_time = self._get_time_from_coverage_field(
853  self.attributes['time_coverage_start'])
854  if 'End Year' in self.attributes:
856  self.attributes['End Year'],
857  self.attributes['End Day'],
858  self.attributes['End Millisec'])
859  elif 'time_coverage_end' in self.attributes:
860  end_time = self._get_time_from_coverage_field(
861  self.attributes['time_coverage_end'])
862  return start_time, end_time
863 
864  def _get_l1_octs_times(self):
865  """
866  Finds and returns timestamps for OCTS L1 data files.
867  """
868  if 'Start Time' in self.attributes:
869  start_time = self._create_octs_l1_timestamp(
870  self.attributes['Start Time'])
871  end_time = self._create_octs_l1_timestamp(
872  self.attributes['End Time'])
873  else:
874  start_time = get_timestamp_from_month_day(''.join([
875  self.attributes['time_coverage_start'][0:4],
876  self.attributes['time_coverage_start'][5:7],
877  self.attributes['time_coverage_start'][8:10]
878  ]), ''.join([
879  self.attributes['time_coverage_start'][11:13],
880  self.attributes['time_coverage_start'][14:16],
881  self.attributes['time_coverage_start'][17:19]
882  ]))
883  end_time = get_timestamp_from_month_day(''.join([
884  self.attributes['time_coverage_end'][0:4],
885  self.attributes['time_coverage_end'][5:7],
886  self.attributes['time_coverage_end'][8:10]
887  ]), ''.join([
888  self.attributes['time_coverage_end'][11:13],
889  self.attributes['time_coverage_end'][14:16],
890  self.attributes['time_coverage_end'][17:19]
891  ]))
892  return start_time, end_time
893 
894  def _get_l1_sgli_times(self):
895  """
896  Finds and returns timestamps for SGLI L1 data files.
897  """
898  if 'Scene_start_time' in self.attributes:
899  start_time = self._create_octs_l1_timestamp(
900  self.attributes['Scene_start_time'])
901  end_time = self._create_octs_l1_timestamp(
902  self.attributes['Scene_end_time'])
903  return start_time, end_time
904 
905  def _get_l1_times(self):
906  """
907  Determines the times for Level 1 files.
908  """
909  start_time = None
910  end_time = None
911  if self.instrument.find('SeaWiFS')!= -1 or \
912  self.instrument.find('Aquarius') != -1 or \
913  self.instrument.find('CZCS') != -1 or \
914  self.instrument.find('MOS') != -1 or \
915  self.instrument.find('OCI') != -1 or \
916  self.instrument.find('OCIS') != -1 or \
917  self.instrument.find('OSMI') != -1 or \
918  self.instrument.find('VIIRS') != -1:
919  if 'Start Time' in self.attributes:
920  start_time = self.attributes['Start Time'][0:13]
921  elif 'time_coverage_start' in self.attributes:
922  start_time = get_timestamp_from_month_day(''.join([
923  self.attributes['time_coverage_start'][0:4],
924  self.attributes['time_coverage_start'][5:7],
925  self.attributes['time_coverage_start'][8:10]
926  ]), ''.join([
927  self.attributes['time_coverage_start'][11:13],
928  self.attributes['time_coverage_start'][14:16],
929  self.attributes['time_coverage_start'][17:19]
930  ]))
931  if 'End Time' in self.attributes:
932  end_time = self.attributes['End Time'][0:13]
933  elif 'time_coverage_end' in self.attributes:
934  end_time = get_timestamp_from_month_day(''.join([
935  self.attributes['time_coverage_end'][0:4],
936  self.attributes['time_coverage_end'][5:7],
937  self.attributes['time_coverage_end'][8:10]
938  ]), ''.join([
939  self.attributes['time_coverage_end'][11:13],
940  self.attributes['time_coverage_end'][14:16],
941  self.attributes['time_coverage_end'][17:19]
942  ]))
943  elif self.instrument.find('HAWKEYE') != -1:
944  if 'time_coverage_start' in self.attributes:
945  start_time = get_timestamp_from_month_day(''.join([
946  self.attributes['time_coverage_start'][0:4],
947  self.attributes['time_coverage_start'][5:7],
948  self.attributes['time_coverage_start'][8:10]
949  ]), ''.join([
950  self.attributes['time_coverage_start'][11:13],
951  self.attributes['time_coverage_start'][14:16],
952  str(round(float(self.attributes['time_coverage_start'][17:23])))
953  ]))
954  if 'time_coverage_end' in self.attributes:
955  end_time = get_timestamp_from_month_day(''.join([
956  self.attributes['time_coverage_end'][0:4],
957  self.attributes['time_coverage_end'][5:7],
958  self.attributes['time_coverage_end'][8:10]
959  ]), ''.join([
960  self.attributes['time_coverage_end'][11:13],
961  self.attributes['time_coverage_end'][14:16],
962  str(round(float(self.attributes['time_coverage_start'][17:23])))
963  ]))
964  elif self.instrument.find('MODIS') != -1:
965  start_time, end_time = self._get_l1_modis_times()
966  elif self.instrument.find('OCTS') != -1:
967  start_time, end_time = self._get_l1_octs_times()
968  elif self.instrument.find('MERIS') != -1:
969  start_time, end_time = self._get_l1_meris_times()
970  elif self.instrument.find('OLCI') != -1:
971  start_time, end_time = self._get_l1_meris_times()
972  elif self.instrument.find('OCM2') != -1:
973  start_time, end_time = self._get_l1_ocm2_times()
974  elif self.instrument.find('HICO') != -1:
975  start_time, end_time = self._get_l1_hico_times()
976  elif self.instrument.find('GOCI') != -1:
977  start_time, end_time = self._get_l1_goci_times()
978  elif self.instrument.find('MSI') != -1:
979  start_time, end_time = self._get_l1_msi_times()
980  elif self.instrument.find('OLI') != -1:
981  start_time, end_time = self._get_landsat_times()
982  elif self.instrument.find('SGLI') != -1:
983  start_time, end_time = self._get_l1_sgli_times()
984  return start_time, end_time
985 
986  def _get_type_using_l0_cnst(self):
987  """
988  Determines the type info based on results of l0cnst_write_modis.
989  """
990  l0cnst_results = self._get_l0_constructor_data()
991  if l0cnst_results is not None:
992  self.l0_data = l0cnst_results
993  self.file_type = 'Level 0'
994  self.instrument = 'MODIS'
995  if re.search(r'^[aA]|^MOD00.P|^MYD\S+.A|^P1540064',
996  os.path.basename(self.file_path)):
997  self.instrument = ' '.join([self.instrument, 'Aqua'])
998  elif re.search(r'^[tT]|^MOD\S+.A|^P0420064',
999  os.path.basename(self.file_path)):
1000  self.instrument = ' '.join([self.instrument, 'Terra'])
1001 
1002  def _get_type_using_short_name(self):
1003  """
1004  Determines the type based on the short name.
1005  """
1006  self.instrument = self.attributes['ASSOCIATEDINSTRUMENTSHORTNAME']
1007  if 'ASSOCIATEDPLATFORMSHORTNAME' in self.attributes:
1008  self.instrument = ' '.join([self.instrument,
1009  self.attributes['ASSOCIATEDPLATFORMSHORTNAME']])
1010  if 'LONGNAME' in self.attributes:
1011  if self.attributes['LONGNAME'].find('Geolocation') != -1:
1012  self.file_type = 'GEO'
1013  elif self.attributes['LONGNAME'].find('L1A') != -1:
1014  self.file_type = 'Level 1A'
1015  elif self.attributes['LONGNAME'].find('L1B') != -1:
1016  self.file_type = 'Level 1B'
1017 
1018  def _get_type_using_title(self):
1019  """
1020  Determines the type based on the title field.
1021  """
1022  if 'Title' in self.attributes:
1023  title = self.attributes['Title']
1024  else:
1025  title = self.attributes['title']
1026  if title.find('OLCI Level 1b') != -1:
1027  self.file_type = 'Level 1B'
1028  if 'product_name' in self.attributes:
1029  product_name = self.attributes['product_name']
1030  if '3A_OL_1_E' in product_name:
1031  self.instrument = 'OLCI S3A'
1032  elif '3B_OL_1_E' in product_name:
1033  self.instrument = 'OLCI S3B'
1034  elif (title.find('Level-1') != -1) or (title.find('Level 1') != -1) or \
1035  (title.find('L1') != -1):
1036  self.file_type, self.instrument = \
1037  self._get_data_from_l1_attributes(title)
1038  elif title.find('Level-2') != -1 or title.find('Level 2') != -1:
1039  self.file_type, self.instrument = \
1041  elif (title.find('Level-3') != -1) or (title.find('level-3') != -1):
1042  self.file_type, self.instrument = \
1044  elif title.find('Level-0') != -1:
1045  self.file_type, self.instrument = \
1047  elif title.find('Ancillary') != -1:
1048  self.file_type, self.instrument = \
1050  elif title.find('VIIRS') != -1:
1051  self.instrument = 'VIIRS'
1052  if 'processing_level' in self.attributes:
1053  if self.attributes['processing_level'].upper().find('L1A') != -1:
1054  self.file_type = 'Level 1A'
1055  elif self.attributes['processing_level'].upper().find('L1B') != -1:
1056  self.file_type = 'Level 1B'
1057  elif self.attributes['processing_level'].upper().find('GEO') != -1:
1058  self.file_type = 'GEO'
1059  elif title.find('L1A') != -1:
1060  self.file_type = 'Level 1A'
1061  elif title.find('L1B') != -1:
1062  self.file_type = 'Level 1B'
1063 
1064  def _read_metadata(self):
1065  """
1066  Using the MetaUtils.readMetadata function, reads the metadata and loads
1067  it into the attributes member variable.
1068  """
1069  try:
1070  if os.path.islink(self.file_path):
1071  self.file_path = os.path.realpath(self.file_path)
1073  if not (attrs is None):
1074  self.attributes = attrs
1075  except SystemExit:
1076  # readMetadata calls sys.exit() when it can't find attributes
1077  # to process, but we don't want to exit here.
1078  pass
1079  except TypeError:
1080  pass
def get_timestamp_from_year_day_mil(year, doy, millisecs)
def _create_timestamp_using_YMDTHMS(self, time_str)
def _create_modis_l1_timestamp(self, rng_date, rng_time)
def readMetadata(filename)
Definition: MetaUtils.py:201
def convert_millisecs_to_time_str(millisecs)
def convert_month_day_to_doy(mon, dom, yr)
Definition: time_utils.py:8
void print(std::ostream &stream, const char *format)
Definition: PrintDebug.hpp:38
def _create_timestamp_using_mon_abbr(self, time_str)
def get_timestamp_from_month_day(sdate, stime)
def _get_time_from_coverage_field(self, coverage_time)
def _get_landsat_end_time(self, start_yr, start_doy, start_hr, start_min, start_sec)
Definition: aerosol.c:136