Classes
class	ProcessorConfig

class	Sensor

class	Sensor_goci

class	Sensor_hawkeye

class	Sensor_meris

class	Sensor_modis

class	Sensor_seawifs

class	Sensor_viirs

Functions
def	get_obpg_data_file_object (file_specification)

def	build_executable_path (prog_name)

def	build_file_list_file (filename, file_list)

def	build_l2gen_par_file (par_contents, input_file, geo_file, output_file)

def	check_options (options)

def	clean_files (delete_list)

def	create_levels_list (rules_sets)

def	create_help_message (rules_sets)

def	do_processing (sensors_sets, par_file, cmd_line_ifile=None)

def	execute_command (command)

def	extract_par_section (par_contents, section)

def	find_geo_file (inp_file)

def	find_geo_file2 (inp_file, instrument, lvl)

def	find_viirs_geo_file (proc, first_svm_file)

def	get_batch_output_name (file_set, suffix)

def	get_data_file_option (par_contents, opt_text)

def	get_extract_params (proc)

def	get_file_date (filename)

def	get_file_handling_opts (par_contents)

def	get_input_files (par_data)

def	get_input_files_type_data (input_files_list)

def	get_intermediate_processors (sensor, existing_procs, rules, lowest_source_level)

def	get_intermediate_products (existing_prod_names, ruleset, lowest_source_level)

def	get_l2_extension ()

def	get_l3bin_extension ()

def	get_lowest_source_level (source_files)

def	get_options (par_data)

def	get_output_name2 (inp_files, targ_prog, suite=None, oformt=None, res=None)

def	get_output_name3 (input_name, input_files, suffix)

def	get_par_file_contents (par_file, acceptable_single_keys)

def	get_processors2 (sensor, par_contents, rules, lowest_source_level)

def	exe_processor (proc, src_files, src_lvl)

def	get_processors (src_files, input_files, par_contents, files_to_delete)

def	get_required_programs (target_program, ruleset, lowest_source_level)

def	get_source_geo_files (source_files, proc_src_types, proc_src_ndx)

def	get_source_file_sets (proc_src_types, source_files, src_key, requires_all_sources)

def	get_source_files (input_files)

def	get_source_products_types (targt_prod, ruleset)

def	get_traceback_message ()

def	initialze_sensors ()

def	log_and_exit (error_msg)

def	main ()

def	process_command_line (cl_parser)

def	read_file_list_file (flf_name)

def	run_batch_processor (processor, file_set)

def	run_nonbatch_processor (processor)

def	run_script (proc, script_name)

def	start_logging (time_stamp)

def	uniqify_list (orig_list)

Variables
bool	DEBUG = False

	cfg_data = None

list	FILE_USE_OPTS = ['deletefiles', 'overwrite', 'use_existing', 'combine_files']

dictionary	SUFFIXES

dictionary	input_file_data = {}

	OCSSWROOT_DIR = os.environ['OCSSWROOT']

Detailed Description

Program to perform multilevel processing (previously known as the
seadas_processor and sometimes referred to as the 'uber' processor).

Function Documentation

◆ build_executable_path()

def multilevel_processor.build_executable_path ( prog_name )

Returns the directory in which the program named in prog_name is found.
None is returned if the program is not found.

Definition at line 821 of file multilevel_processor.py.

◆ build_file_list_file()

def multilevel_processor.build_file_list_file	(	filename,
		file_list
	)

Create a file listing the names of the files to be processed.

Definition at line 835 of file multilevel_processor.py.

◆ build_l2gen_par_file()

def multilevel_processor.build_l2gen_par_file	(	par_contents,
		input_file,
		geo_file,
		output_file
	)

Build the parameter file for L2 processing.

Definition at line 843 of file multilevel_processor.py.

◆ check_options()

def multilevel_processor.check_options ( options )

Check command line options

Definition at line 863 of file multilevel_processor.py.

◆ clean_files()

def multilevel_processor.clean_files ( delete_list )

Delete unwanted files created during processing.

Definition at line 878 of file multilevel_processor.py.

◆ create_help_message()

def multilevel_processor.create_help_message ( rules_sets )

Creates the message to be displayed when help is provided.

Definition at line 940 of file multilevel_processor.py.

◆ create_levels_list()

def multilevel_processor.create_levels_list ( rules_sets )

Returns a list containing all the levels from all the rules sets.

Definition at line 918 of file multilevel_processor.py.

◆ do_processing()

def multilevel_processor.do_processing	(	sensors_sets,
		par_file,
		cmd_line_ifile = `None`
	)

Perform the processing for each step (element of processor_list) needed.

Definition at line 995 of file multilevel_processor.py.

◆ exe_processor()

def multilevel_processor.exe_processor	(	proc,
		src_files,
		src_lvl
	)

Execute the processor.

Definition at line 1580 of file multilevel_processor.py.

◆ execute_command()

def multilevel_processor.execute_command ( command )

Execute what is contained in command and then output the results to log
files and the console, as appropriate.

Definition at line 1083 of file multilevel_processor.py.

◆ extract_par_section()

def multilevel_processor.extract_par_section	(	par_contents,
		section
	)

Returns a single section (e.g. L1a, GEO, L1B, L2, etc.) from the "par" file.

Definition at line 1104 of file multilevel_processor.py.

◆ find_geo_file()

def multilevel_processor.find_geo_file ( inp_file )

Searches for a GEO file corresponding to inp_file.  If that GEO file exists,
returns that file name; otherwise, returns None.

Definition at line 1113 of file multilevel_processor.py.

◆ find_geo_file2()

def multilevel_processor.find_geo_file2	(	inp_file,
		instrument,
		lvl
	)

Searches for a GEO file corresponding to inp_file.  If that GEO file exists,
returns that file name; otherwise, returns None.

Definition at line 1128 of file multilevel_processor.py.

◆ find_viirs_geo_file()

def multilevel_processor.find_viirs_geo_file	(	proc,
		first_svm_file
	)

Searches for a GEO file corresponding to first_svm_file.  If that GEO file
exists, returns that file name; otherwise, returns None.

Definition at line 1165 of file multilevel_processor.py.

◆ get_batch_output_name()

def multilevel_processor.get_batch_output_name	(	file_set,
		suffix
	)

Returns the output file for a "batch" run, i.e. a process that can accept
multiple inputs, such as l2bin or l3bin.

Definition at line 1175 of file multilevel_processor.py.

◆ get_data_file_option()

def multilevel_processor.get_data_file_option	(	par_contents,
		opt_text
	)

If found in par_contents, the value for the option specified by opt_text
is returned; otherwise, False is returned.

Definition at line 1213 of file multilevel_processor.py.

◆ get_extract_params()

def multilevel_processor.get_extract_params ( proc )

Run the lonlat2pixline program and return the parameters found.

Definition at line 1228 of file multilevel_processor.py.

◆ get_file_date()

def multilevel_processor.get_file_date ( filename )

Get a Python Date object from a recognized file name's year and day of year.

Definition at line 1265 of file multilevel_processor.py.

◆ get_file_handling_opts()

def multilevel_processor.get_file_handling_opts ( par_contents )

Returns the values of the file handling options in par_contents.

Definition at line 1288 of file multilevel_processor.py.

◆ get_input_files()

def multilevel_processor.get_input_files ( par_data )

Get input files found in the uber par file's ifile line, a file list file,
or both.  Ensure that the list contains no duplicates.

Definition at line 1298 of file multilevel_processor.py.

◆ get_input_files_type_data()

def multilevel_processor.get_input_files_type_data ( input_files_list )

Returns a dictionary with the the file_type (L0, L1A, L2, etc) and
instrument for each file in the input list.

Definition at line 1325 of file multilevel_processor.py.

◆ get_intermediate_processors()

def multilevel_processor.get_intermediate_processors	(	sensor,
		existing_procs,
		rules,
		lowest_source_level
	)

Create processor objects for products which are needed, but not explicitly
specified in the par file.

Definition at line 1367 of file multilevel_processor.py.

◆ get_intermediate_products()

def multilevel_processor.get_intermediate_products	(	existing_prod_names,
		ruleset,
		lowest_source_level
	)

Find products which are needed, but not explicitly specified by the
par file.

Definition at line 1385 of file multilevel_processor.py.

◆ get_l2_extension()

def multilevel_processor.get_l2_extension ( )

Returns the extension for an L2 file.  For the time being, this is
just '.L2'; however, different extensions may be wanted in the future, thus
this function is in place.

Definition at line 1402 of file multilevel_processor.py.

◆ get_l3bin_extension()

def multilevel_processor.get_l3bin_extension ( )

Returns the extension for an L3 Binned file.  For the time being, this is
just '.L3bin'; however, different extensions may be wanted in the future,
thus this function is in place.

Definition at line 1410 of file multilevel_processor.py.

◆ get_lowest_source_level()

def multilevel_processor.get_lowest_source_level ( source_files )

Find the level of the lowest level source file to be processed.

Definition at line 1418 of file multilevel_processor.py.

◆ get_obpg_data_file_object()

def multilevel_processor.get_obpg_data_file_object ( file_specification )

Returns an obpg_data_file object for the file named in file_specification.

Definition at line 809 of file multilevel_processor.py.

◆ get_options()

def multilevel_processor.get_options ( par_data )

Extract the options for a program to be run from the corresponding data in
the uber par file.

Definition at line 1434 of file multilevel_processor.py.

◆ get_output_name2()

def multilevel_processor.get_output_name2	(	inp_files,
		targ_prog,
		suite = `None`,
		oformt = `None`,
		res = `None`
	)

Determine what the output name would be if targ_prog is run on input_files.

Definition at line 1448 of file multilevel_processor.py.

◆ get_output_name3()

def multilevel_processor.get_output_name3	(	input_name,
		input_files,
		suffix
	)

Determine the output name for a program to be run.

Definition at line 1465 of file multilevel_processor.py.

◆ get_par_file_contents()

def multilevel_processor.get_par_file_contents	(	par_file,
		acceptable_single_keys
	)

Return the contents of the input "par" file.

Definition at line 1507 of file multilevel_processor.py.

◆ get_processors()

def multilevel_processor.get_processors	(	src_files,
		input_files,
		par_contents,
		files_to_delete
	)

Determine how to chain the processors together.

Definition at line 1612 of file multilevel_processor.py.

◆ get_processors2()

def multilevel_processor.get_processors2	(	sensor,
		par_contents,
		rules,
		lowest_source_level
	)

Determine the processors which are needed.

Definition at line 1562 of file multilevel_processor.py.

◆ get_required_programs()

def multilevel_processor.get_required_programs	(	target_program,
		ruleset,
		lowest_source_level
	)

Returns the programs required too produce the desired final output.

Definition at line 2004 of file multilevel_processor.py.

◆ get_source_file_sets()

def multilevel_processor.get_source_file_sets	(	proc_src_types,
		source_files,
		src_key,
		requires_all_sources
	)

Returns the set of source files needed.

Definition at line 2047 of file multilevel_processor.py.

◆ get_source_files()

def multilevel_processor.get_source_files ( input_files )

Returns a dictionary containing the programs to be run (as keys) and the
a list of files on which that program should be run.

Definition at line 2106 of file multilevel_processor.py.

◆ get_source_geo_files()

def multilevel_processor.get_source_geo_files	(	source_files,
		proc_src_types,
		proc_src_ndx
	)

:param source_files: list of source files
:param proc_src_types: list of source types for the processor
:param proc_src_ndx: index into the proc_src_types list pointing to the
                     source type to use to get the input files
:return: list of GEO files that correspond to the files in source_files

Definition at line 2027 of file multilevel_processor.py.

◆ get_source_products_types()

def multilevel_processor.get_source_products_types	(	targt_prod,
		ruleset
	)

Return the list of source product types needed to produce the final product.

Definition at line 2120 of file multilevel_processor.py.

◆ get_traceback_message()

def multilevel_processor.get_traceback_message ( )

Returns an error message built from traceback data.

Definition at line 2135 of file multilevel_processor.py.

◆ initialze_sensors()

def multilevel_processor.initialze_sensors ( )

Initialize sensors.

Definition at line 2152 of file multilevel_processor.py.

◆ log_and_exit()

def multilevel_processor.log_and_exit ( error_msg )

Record error_msg in the debug log, then exit with error_msg going to stderr
and an exit code of 1; see:
    http://docs.python.org/library/sys.html#exit.

Definition at line 2165 of file multilevel_processor.py.

◆ main()

def multilevel_processor.main ( )

main processing function.

Definition at line 2174 of file multilevel_processor.py.

◆ process_command_line()

def multilevel_processor.process_command_line ( cl_parser )

Get arguments and options from the calling command line.
To be consistent with other OBPG programs, an underscore ('_') is used for
multiword options, instead of a dash ('-').

Definition at line 2239 of file multilevel_processor.py.

◆ read_file_list_file()

def multilevel_processor.read_file_list_file ( flf_name )

Reads flf_name and returns the list of files to be processed.

Definition at line 2282 of file multilevel_processor.py.

◆ run_batch_processor()

def multilevel_processor.run_batch_processor	(	processor,
		file_set
	)

Run a processor, e.g. l2bin, which processes batches of files.

Definition at line 2303 of file multilevel_processor.py.

◆ run_nonbatch_processor()

def multilevel_processor.run_nonbatch_processor ( processor )

Run a processor which deals with single input files (or pairs of files in
the case of MODIS L1B processing in which GEO files are also needed).

Definition at line 2351 of file multilevel_processor.py.

◆ run_script()

def multilevel_processor.run_script	(	proc,
		script_name
	)

Build the command to run the processing script which is passed in.

Definition at line 2427 of file multilevel_processor.py.

◆ start_logging()

def multilevel_processor.start_logging ( time_stamp )

Opens log file(s) for debugging.

Definition at line 2442 of file multilevel_processor.py.

◆ uniqify_list()

def multilevel_processor.uniqify_list ( orig_list )

Returns a list with no duplicates.  Somewhat borrowed from:
  http://www.peterbe.com/plog/uniqifiers-benchmark (example f5)

Definition at line 2465 of file multilevel_processor.py.

Variable Documentation

◆ cfg_data

cfg_data = None

Definition at line 2483 of file multilevel_processor.py.

◆ DEBUG

bool DEBUG = False

Definition at line 2480 of file multilevel_processor.py.

◆ FILE_USE_OPTS

list FILE_USE_OPTS = ['deletefiles', 'overwrite', 'use_existing', 'combine_files']

Definition at line 2484 of file multilevel_processor.py.

◆ input_file_data

dictionary input_file_data = {}

Definition at line 2504 of file multilevel_processor.py.

◆ OCSSWROOT_DIR

OCSSWROOT_DIR = os.environ['OCSSWROOT']

Definition at line 2508 of file multilevel_processor.py.

◆ SUFFIXES

dictionary SUFFIXES

Initial value:

 =  {
     'geo': 'GEO',
     'l1brsgen': 'L1B_BRS',
     'l1aextract': 'L1A.sub',
     'l1aextract_viirs': 'L1A.sub',
     'l1aextract_seawifs': 'L1A.sub',
     'l1aextract_modis': 'L1A.sub',
     # 'l1mapgen': 'L1B_MAP',
     'l2bin': 'L3b',
     'l2brsgen': 'L2_BRS',
     'l2extract': 'L2.sub',
     'l2gen': 'L2',
     # 'l2mapgen': 'L2B_MAP',
     #'l3bin': 'L3b',
     'l3mapgen': 'L3m',
     'level 1a': 'L1A',
     'level 1b': 'L1B_LAC',
     # 'smigen': 'SMI'
 }

Definition at line 2485 of file multilevel_processor.py.

Classes

Functions

Variables