3 A class for determining the OBPG type of a file.
7 __author__ =
'melliott'
23 KNOWN_SENSORS = [
'Aquarius',
'CZCS',
'HAWKEYE',
'HICO',
24 'HMODISA',
'HMODIST',
'MERIS',
'MODISA',
25 'MODIS Aqua',
'MODIST',
'MODIS Terra',
26 'MOS',
'OCI',
'OCIS',
'OCM2',
'OCTS',
'OSMI',
27 'SeaWiFS',
'SGLI',
'VIIRSN',
'VIIRSJ1',
'VIIRSJ2']
29 MONTH_ABBRS = dict((v.upper(), k)
for k, v
in enumerate(calendar.month_abbr))
34 Converts a number of milliseconds to a string representing the time of day.
36 secs = math.trunc(millisecs / 1000)
37 hrs = math.trunc(secs / 3600)
38 secs = secs - (hrs * 3600)
39 mins = math.trunc(secs / 60)
40 secs = secs - (mins * 60)
41 tm_str =
'{0:02}{1:02}{2:02}'.format(hrs, mins, secs)
46 Creates timestamp for VIIRS L1.
48 year =
int(sdate[0:4])
52 mins =
int(stime[2:4])
53 secs =
int(stime[4:6])
54 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
55 return dt_obj.strftime(
'%Y%j%H%M%S')
59 Return a timestamp in the YYYYDDDHHMMSS form from a given year (yr), day
60 of year (doy) and milliseconds (after 00:00 - millisecs).
63 timestamp =
'{0:04}{1:03}{2}'.format(year, doy, time_str)
68 A class containing methods for finding the type of an OBPG file (e.g.
69 MODIS L2b, SeaWiFS L1A, Aquarius L3m, etc.
74 Save the path of the file in question and set up default
75 values for several thing still to be found.
77 if os.path.exists(fpath):
78 if tarfile.is_tarfile(fpath):
82 err_msg =
"Error! Cannot process file {0}.".format(fpath)
93 err_msg =
"Error! File {0} could not be found.".format(fpath)
109 def _create_modis_l1_timestamp(self, rng_date, rng_time):
111 Returns a date/time stamp for a MODIS L1 file from the appropriate
112 RANGINGDATE and RANGINGTIME attributes. The returned date/time stamp
113 is of form YYYYDDDHHMMSS, where YYYY = year, DDD = day of year,
114 HH = hour, MM = minute, and SS = second.
116 year =
int(rng_date[0:4])
117 mon =
int(rng_date[5:7])
118 dom =
int(rng_date[8:10])
119 hrs =
int(rng_time[0:2])
120 mins =
int(rng_time[3:5])
121 secs =
int(rng_time[6:8])
122 milsecs =
int(rng_time[9:15])
123 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs, milsecs)
124 if milsecs >= 500000:
125 dt_delta = datetime.timedelta(seconds=1)
127 return dt_obj.strftime(
'%Y%j%H%M%S')
129 def _create_octs_l1_timestamp(self, dt_str):
131 Creates a timestamp for an OCTS L1.
133 year =
int(dt_str[0:4])
134 mon =
int(dt_str[4:6])
135 dom =
int(dt_str[6:8])
136 hrs =
int(dt_str[9:11])
137 mins =
int(dt_str[12:14])
138 secs =
int(dt_str[15:17])
139 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
140 return dt_obj.strftime(
'%Y%j%H%M%S')
142 def _create_timestamp_using_mon_abbr(self, time_str):
144 Returns a properly formatted date/time stamp for MERIS L1B files from
145 an attribute in the file.
148 year =
int(time_str[7:11])
149 mon =
int(MONTH_ABBRS[time_str[3:6]])
150 dom =
int(time_str[0:2])
151 hrs =
int(time_str[12:14])
152 mins =
int(time_str[15:17])
153 secs =
int(time_str[18:20])
154 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
155 return dt_obj.strftime(
'%Y%j%H%M%S')
157 def _create_timestamp_using_YMDTHMS(self, time_str):
159 Returns a properly formatted date/time stamp for MERIS L1B files from
160 an attribute in the file.
163 year =
int(time_str[0:4])
164 mon =
int(time_str[5:7])
165 dom =
int(time_str[8:10])
166 hrs =
int(time_str[11:13])
167 mins =
int(time_str[14:16])
168 secs =
int(time_str[17:19])
169 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
170 return dt_obj.strftime(
'%Y%j%H%M%S')
172 def _extract_viirs_sdr(self, tar_path):
176 tar_file = tarfile.TarFile(tar_path)
177 tar_members = tar_file.getnames()
178 for mbr
in tar_members:
179 if mbr.startswith(
'SVM01'):
180 mbr_info = tar_file.getmember(mbr)
181 tar_file.extractall(members=[mbr_info], path=
'/tmp')
182 self.
file_path = os.path.join(
'/tmp', mbr)
186 exc_info = sys.exc_info()
187 for item
in exc_info:
193 def _get_data_from_anc_attributes(self):
195 Processes Ancillary data files.
197 instrument =
'Ancillary'
199 return file_type, instrument
201 def _get_data_from_l0_attributes(self):
203 Get the instrument and file type from the attributes for an L0 file.
205 file_type =
'unknown'
208 if title_parts[1].find(
'Level-0') != -1:
209 file_type =
'Level 0'
210 instrument = title_parts[0].strip()
211 return file_type, instrument
213 def _get_data_from_l1_attributes(self, title):
215 Get the instrument and file type from the attributes for an L1 file.
217 file_type =
'Level 1'
218 instrument =
'unknown'
219 possible_levels = {
'Level 1A':
'Level 1A',
'Level-1A':
'Level 1A',
220 'L1A':
'Level 1A',
'L1a':
'Level 1A',
221 'Level 1B':
'Level 1B',
'Level-1B':
'Level 1B',
222 'L1B':
'Level 1B',
'L1b':
'Level 1B'}
224 if title.find(
'Level 1') != -1:
225 working_title = title.replace(
'Level 1',
'Level-1')
227 working_title = title
228 working_title = re.sub(
"""^['"](.*)['"].*;$""",
'\g<1>', working_title)
229 title_parts = working_title.split()
230 for part
in title_parts:
231 if part
in KNOWN_SENSORS:
235 for ks
in KNOWN_SENSORS:
236 if part.find(ks) != -1:
240 if part
in possible_levels:
241 file_type = possible_levels[part]
242 if title.find(
'Browse Data') != -1:
243 file_type +=
' Browse Data'
244 return file_type, instrument
246 def _get_data_from_l2_attributes(self):
248 Get the instrument and file type from the attributes for an 20 file.
250 file_type =
'Level 2'
252 title_parts = self.
attributes[
'Title'].split()
253 if self.
attributes[
'Title'].find(
'Browse Data') != -1:
254 file_type +=
' Browse Data'
256 title_parts = self.
attributes[
'title'].split()
257 if self.
attributes[
'title'].find(
'Browse Data') != -1:
258 file_type +=
' Browse Data'
259 instrument = title_parts[0].strip()
260 return file_type, instrument
262 def _get_data_from_l3_attributes(self):
264 Get the instrument and file type from the attributes for an L3 file.
266 file_type =
'unknown'
273 if (working_title.find(
'Level-3 Binned') != -1)
or \
274 (working_title.find(
'level-3_binned') != -1):
275 file_type =
'Level 3 Binned'
276 elif working_title.find(
'Level-3 Standard Mapped Image') != -1:
277 file_type =
'Level 3 SMI'
279 return file_type, instrument
281 def _get_instrument(self):
283 Get the instrument from the attributes.
285 instrument =
'unknown'
288 title_parts = self.
attributes[
'Title'].split()
290 title_parts = self.
attributes[
'title'].split()
291 if title_parts[0].find(
'Level') == -1:
292 instrument = title_parts[0].strip()
295 if not (instrument
in KNOWN_SENSORS):
296 if instrument !=
'unknown':
298 instrument =
' '.join([instrument,
302 instrument = self.
attributes[
'Sensor Name'].strip()
304 (instrument
in KNOWN_SENSORS):
305 instrument = self.
attributes[
'Sensor'].strip()
308 def _get_l0_constructor_data(self):
310 Try to get data from the MODIS L0 constructor for this file.
312 l0cnst_results =
None
313 cmd = [
'l0cnst_write_modis',
'-n', self.
file_path]
315 l0_out = subprocess.Popen(cmd, shell=
False, stdout=subprocess.PIPE,
316 stderr=subprocess.PIPE).communicate()[0]
317 out_lines = l0_out.decode(
'utf-8').split(
'\n')
319 for line
in out_lines:
320 if line.find(
'Total packets') != -1:
322 packets_read =
float(parts[0])
323 if packets_read > 0.0:
328 if starttime
and stoptime:
329 l0cnst_results = {
'starttime' : starttime,
330 'stoptime' : stoptime}
332 l0cnst_results =
None
333 return l0cnst_results
335 def _get_l0_times(self):
337 Returns the start and end times for L0 data files.
339 if isinstance(self.
l0_data, dict):
340 start_time = self.
l0_data[
'starttime']
341 end_time = self.
l0_data[
'stoptime']
344 if line.find(
'starttime') != -1:
345 time_part = line.split(
'=')[1]
346 start_time = time_part[0:4] + time_part[5:7] + \
347 time_part[8:10] + time_part[11:13] + \
348 time_part[14:16] + time_part[17:19]
349 if line.find(
'stoptime') != -1:
350 time_part = line.split(
'=')[1]
351 end_time = time_part[0:4] + time_part[5:7] + \
352 time_part[8:10] + time_part[11:13] +\
353 time_part[14:16] + time_part[17:19]
354 return start_time, end_time
356 def _get_landsat_end_time(self, start_yr, start_doy, start_hr, start_min,
359 Compute and return the end time for a Landsat OLI data file, given the
360 start year, day of year, hour, mine, and second.
364 end_hr =
int(start_hr)
365 end_min =
int(start_min)
366 end_sec =
int(start_sec) + 1
383 end_time =
'{0:4d}{1:03d}{2:02d}{3:02d}{4:02d}'.format(end_yr,
390 def _get_landsat_times(self):
392 Computes and returns the start and end times for a Landsat OLI
398 acquisition_date = self.
attributes[
'DATE_ACQUIRED'].strip(
'"')
399 center_time = self.
attributes[
'SCENE_CENTER_TIME'].strip(
'"')
401 start_yr =
int(acquisition_date[0:4])
402 start_mon =
int(acquisition_date[5:7])
403 start_dom =
int(acquisition_date[8:10])
406 start_date_str =
'{0:04d}{1:03d}'.format(start_yr, start_doy)
407 start_hr = center_time[0:2]
408 start_min = center_time[3:5]
409 start_sec = center_time[6:8]
410 start_time =
''.join([start_date_str, start_hr, start_min,
413 start_hr, start_min, start_sec)
414 return start_time, end_time
416 def _get_time_from_coverage_field(self, coverage_time):
418 Returns the stamp computed from the coverage_time
420 if coverage_time[0:4] ==
'':
423 yr =
'{0:04d}'.format(
int(coverage_time[0:4]))
424 doy =
'{0:03d}'.format(
428 if coverage_time[11:13] ==
'':
431 hr =
'{0:02d}'.format(
int(coverage_time[11:13]))
432 if coverage_time[14:16] ==
'':
435 min =
'{0:02d}'.format(
int(coverage_time[14:16]))
436 if coverage_time[17:19] ==
'':
439 sec =
'{0:02d}'.format(
int(coverage_time[17:19]))
440 time_stamp =
''.join([yr,
str(doy), hr, min, sec])
446 API method to return the file's attributes.
452 Returns the start and end time for a file.
455 start_time =
'unable to determine start time'
456 end_time =
'unable to determine end time'
464 elif self.
file_type.find(
'Level 1') != -1:
466 elif self.
file_type.find(
'Level 2') != -1
or \
475 start_time = self.
attributes[
'Start Time'][0:13]
476 elif 'time_coverage_start' in self.
attributes:
500 start_time =
'{0:04}{1:03}{2}'.format(
506 end_time =
'{0:04}{1:03}{2}'.format(
512 elif self.
file_type.find(
'Level 0') != -1:
514 return start_time, end_time
516 def _get_type_using_platform(self):
517 levelMap = {
'L0':
'Level 0',
523 'L3 Binned':
'Level 3 Binned',
524 'L3 Mapped':
'Level 3 SMI'
526 instrumentMap = {
'SEAWIFS':
"SeaWiFS",
540 "AQUARIUS":
"Aquarius" ,
547 "HAWKEYE":
"HAWKEYE",
552 if self.
attributes[
'processing_level']
in levelMap:
555 if self.
attributes[
'title'].find(
'Geolocation') != -1:
558 if self.
attributes[
'title'].find(
'Standard Mapped Image') == -1:
561 if inst
in instrumentMap:
565 if inst.find(
'MODIS') != -1:
566 if plat.find(
'AQUA') != -1:
569 elif plat.find(
'TERRA') != -1:
572 elif inst.find(
'VIIRS') != -1:
573 if plat.find(
'SUOMI-NPP') != -1:
576 elif plat.find(
'JPSS-1') != -1:
579 elif plat.find(
'JPSS-2') != -1:
582 elif inst.find(
'OLCI') != -1:
583 if plat ==
'SENTINEL-3' or plat.find(
'3A') != -1:
586 elif plat.find(
'3B') != -1:
589 elif inst.find(
'MSI') != -1:
590 if plat ==
'SENTINEL-2' or plat.find(
'2A') != -1:
593 elif plat.find(
'2B') != -1:
596 elif inst.find(
'OLI') != -1:
597 if plat ==
'LANDSAT-8':
600 elif plat ==
'LANDSAT-9':
605 def _get_file_type_from_attributes(self):
607 Determines the file type and instrument from the file attributes and
608 sets those values in the object.
617 elif 'ASSOCIATEDINSTRUMENTSHORTNAME' in self.
attributes:
620 if self.
attributes[
'PRODUCT'].find(
'MER_') != -1:
623 elif 'Instrument_Short_Name' in self.
attributes:
626 if self.
attributes[
'processing_level'].find(
'L1B') != -1:
633 elif mission ==
'JPSS-1':
641 if not self.
file_type.startswith(
'Level'):
644 if self.
attributes[
'Sensor'].find(
'SGLI') != -1:
652 Returns what type (L1A, L2, etc.) a file is and what
653 platform/sensor/instrument made the observations.
669 def _get_file_type_landsat(self):
671 Sets the file type and instrument for Landsat OLI data files
675 if self.
attributes[
'SPACECRAFT_ID'].find(
'LANDSAT_8') != -1:
683 if self.
attributes[
'DATA_TYPE'].find(
'L1G') != -1
or \
684 self.
attributes[
'DATA_TYPE'].find(
'L1GT') != -1
or \
685 self.
attributes[
'DATA_TYPE'].find(
'L1P') != -1
or \
686 self.
attributes[
'DATA_TYPE'].find(
'L1T') != -1:
689 if self.
attributes[
'PROCESSING_LEVEL'].find(
'L1G') != -1
or \
690 self.
attributes[
'PROCESSING_LEVEL'].find(
'L1GT') != -1
or \
691 self.
attributes[
'PROCESSING_LEVEL'].find(
'L1P') != -1
or \
692 self.
attributes[
'PROCESSING_LEVEL'].find(
'L1T') != -1
or\
693 self.
attributes[
'PROCESSING_LEVEL'].find(
'L1TP') != -1:
700 Returns what type (L1A, L2, etc.) a file is and what
701 platform/sensor/instrument made the observations.
738 def _get_l0_start_stop_times(self, l0_lines):
740 Returns the start time and stop time if found in l0_lines.
744 for line
in l0_lines:
745 if line.find(
'starttime') != -1:
746 starttime = line.strip().split(
'=')[1]
747 if line.find(
'stoptime') != -1:
748 stoptime = line.strip().split(
'=')[1]
749 return starttime, stoptime
751 def _get_l1_goci_times(self):
753 Finds and returns timestamps for GOCI L1 data files.
760 elif 'processing start time' in self.
attributes:
762 mon = time.strptime(self.
attributes[
'processing start time']\
766 start_time =
'{0:04d}{1:03d}{2:02d}{3}{4}'.format(st_yr, doy,
768 self.
attributes[
'processing start time'][15:17],
769 self.
attributes[
'processing start time'][18:20])
774 elif 'processing end time' in self.
attributes:
776 mon = time.strptime(self.
attributes[
'processing end time']\
780 end_time =
'{0:04d}{1:03d}{2:02d}{3}{4}'.format(end_yr, doy,
782 self.
attributes[
'processing end time'][15:17],
783 self.
attributes[
'processing end time'][18:20])
784 return start_time, end_time
786 def _get_l1_hico_times(self):
788 Finds and returns timestamps for HICO L1 data files.
796 return start_time, end_time
798 def _get_l1_meris_times(self):
800 Finds and returns timestamps for MERIS L1 data files.
817 return start_time, end_time
819 def _get_l1_modis_times(self):
821 Finds and returns timestamps for MODIS L1 data files.
829 return start_time, end_time
831 def _get_l1_msi_times(self):
833 Finds and returns timestamps for MSI L1 data files.
838 end_time = start_time
839 return start_time, end_time
841 def _get_l1_ocm2_times(self):
843 Finds and returns timestamps for OCM2 L1 data files.
851 elif 'time_coverage_start' in self.
attributes:
862 return start_time, end_time
864 def _get_l1_octs_times(self):
866 Finds and returns timestamps for OCTS L1 data files.
879 self.
attributes[
'time_coverage_start'][11:13],
880 self.
attributes[
'time_coverage_start'][14:16],
892 return start_time, end_time
894 def _get_l1_sgli_times(self):
896 Finds and returns timestamps for SGLI L1 data files.
903 return start_time, end_time
905 def _get_l1_times(self):
907 Determines the times for Level 1 files.
920 start_time = self.
attributes[
'Start Time'][0:13]
921 elif 'time_coverage_start' in self.
attributes:
927 self.
attributes[
'time_coverage_start'][11:13],
928 self.
attributes[
'time_coverage_start'][14:16],
950 self.
attributes[
'time_coverage_start'][11:13],
951 self.
attributes[
'time_coverage_start'][14:16],
984 return start_time, end_time
986 def _get_type_using_l0_cnst(self):
988 Determines the type info based on results of l0cnst_write_modis.
991 if l0cnst_results
is not None:
995 if re.search(
r'^[aA]|^MOD00.P|^MYD\S+.A|^P1540064',
998 elif re.search(
r'^[tT]|^MOD\S+.A|^P0420064',
1002 def _get_type_using_short_name(self):
1004 Determines the type based on the short name.
1007 if 'ASSOCIATEDPLATFORMSHORTNAME' in self.
attributes:
1009 self.
attributes[
'ASSOCIATEDPLATFORMSHORTNAME']])
1011 if self.
attributes[
'LONGNAME'].find(
'Geolocation') != -1:
1013 elif self.
attributes[
'LONGNAME'].find(
'L1A') != -1:
1015 elif self.
attributes[
'LONGNAME'].find(
'L1B') != -1:
1018 def _get_type_using_title(self):
1020 Determines the type based on the title field.
1026 if title.find(
'OLCI Level 1b') != -1:
1029 product_name = self.
attributes[
'product_name']
1030 if '3A_OL_1_E' in product_name:
1032 elif '3B_OL_1_E' in product_name:
1034 elif (title.find(
'Level-1') != -1)
or (title.find(
'Level 1') != -1)
or \
1035 (title.find(
'L1') != -1):
1038 elif title.find(
'Level-2') != -1
or title.find(
'Level 2') != -1:
1041 elif (title.find(
'Level-3') != -1)
or (title.find(
'level-3') != -1):
1044 elif title.find(
'Level-0') != -1:
1047 elif title.find(
'Ancillary') != -1:
1050 elif title.find(
'VIIRS') != -1:
1053 if self.
attributes[
'processing_level'].upper().find(
'L1A') != -1:
1055 elif self.
attributes[
'processing_level'].upper().find(
'L1B') != -1:
1057 elif self.
attributes[
'processing_level'].upper().find(
'GEO') != -1:
1059 elif title.find(
'L1A') != -1:
1061 elif title.find(
'L1B') != -1:
1064 def _read_metadata(self):
1066 Using the MetaUtils.readMetadata function, reads the metadata and loads
1067 it into the attributes member variable.
1073 if not (attrs
is None):