multilevel_processor2 Namespace Reference

Classes
class	ProcessorConfig
class	Sensor
class	Sensor_goci
class	Sensor_hawkeye
class	Sensor_meris
class	Sensor_modis
class	Sensor_seawifs
class	Sensor_viirs

Functions
def	get_obpg_data_file_object (file_specification)
def	build_executable_path (prog_name)
def	build_file_list_file (filename, file_list)
def	build_l2gen_par_file (par_contents, input_file, geo_file, output_file)
def	check_options (options)
def	clean_files (delete_list)
def	create_levels_list (rules_sets)
def	create_help_message (rules_sets)
def	do_processing (sensors_sets, par_file, cmd_line_ifile=None)
def	execute_command (command)
def	extract_par_section (par_contents, section)
def	find_geo_file (inp_file)
def	find_geo_file2 (inp_file, instrument, lvl)
def	find_viirs_geo_file (proc, first_svm_file)
def	get_batch_output_name (file_set, suffix)
def	get_data_file_option (par_contents, opt_text)
def	get_extract_params (proc)
def	get_file_date (filename)
def	get_file_handling_opts (par_contents)
def	get_input_files (par_data)
def	get_input_files_type_data (input_files_list)
def	get_intermediate_processors (sensor, existing_procs, rules, lowest_source_level)
def	get_intermediate_products (existing_prod_names, ruleset, lowest_source_level)
def	get_l2_extension ()
def	get_l3bin_extension ()
def	get_lowest_source_level (source_files)
def	get_options (par_data)
def	get_output_name2 (inp_files, targ_prog, suite=None, oformt=None, res=None)
def	get_output_name3 (input_name, input_files, suffix)
def	get_par_file_contents (par_file, acceptable_single_keys)
def	get_processors2 (sensor, par_contents, rules, lowest_source_level)
def	exe_processor (proc, src_files, src_lvl)
def	get_processors (src_files, input_files, par_contents, files_to_delete)
def	get_required_programs (target_program, ruleset, lowest_source_level)
def	get_source_geo_files (source_files, proc_src_types, proc_src_ndx)
def	get_source_file_sets (proc_src_types, source_files, src_key, requires_all_sources)
def	get_source_files (input_files)
def	get_source_products_types (targt_prod, ruleset)
def	get_traceback_message ()
def	initialze_sensors ()
def	log_and_exit (error_msg)
def	main ()
def	process_command_line (cl_parser)
def	read_file_list_file (flf_name)
def	run_batch_processor (processor, file_set)
def	run_nonbatch_processor (processor)
def	run_script (proc, script_name)
def	start_logging (time_stamp)
def	uniqify_list (orig_list)

Variables
bool	DEBUG = False
	cfg_data = None
list	FILE_USE_OPTS = ['deletefiles', 'overwrite', 'use_existing']
dictionary	SUFFIXES
dictionary	input_file_data = {}
	OCSSWROOT_DIR = os.environ['OCSSWROOT']

Detailed Description

Program to perform multilevel processing (previously known as the
seadas_processor and sometimes referred to as the 'uber' processor).

Function Documentation

build_executable_path()

def multilevel_processor2.build_executable_path

(

prog_name

)

Returns the directory in which the program named in prog_name is found.
None is returned if the program is not found.

Definition at line 880 of file multilevel_processor2.py.

build_file_list_file()

def multilevel_processor2.build_file_list_file	(		filename,
			file_list
	)

Create a file listing the names of the files to be processed.

Definition at line 894 of file multilevel_processor2.py.

build_l2gen_par_file()

def multilevel_processor2.build_l2gen_par_file	(		par_contents,
			input_file,
			geo_file,
			output_file
	)

Build the parameter file for L2 processing.

Definition at line 902 of file multilevel_processor2.py.

check_options()

def multilevel_processor2.check_options

(

options

)

Check command line options

Definition at line 922 of file multilevel_processor2.py.

clean_files()

def multilevel_processor2.clean_files

(

delete_list

)

Delete unwanted files created during processing.

Definition at line 937 of file multilevel_processor2.py.

create_help_message()

def multilevel_processor2.create_help_message

(

rules_sets

)

Creates the message to be displayed when help is provided.

Definition at line 999 of file multilevel_processor2.py.

create_levels_list()

def multilevel_processor2.create_levels_list

(

rules_sets

)

Returns a list containing all the levels from all the rules sets.

Definition at line 977 of file multilevel_processor2.py.

do_processing()

def multilevel_processor2.do_processing	(		sensors_sets,
			par_file,
			cmd_line_ifile = None
	)

Perform the processing for each step (element of processor_list) needed.

Definition at line 1053 of file multilevel_processor2.py.

exe_processor()

def multilevel_processor2.exe_processor	(		proc,
			src_files,
			src_lvl
	)

Definition at line 1619 of file multilevel_processor2.py.

execute_command()

def multilevel_processor2.execute_command

(

command

)

Execute what is contained in command and then output the results to log
files and the console, as appropriate.

Definition at line 1138 of file multilevel_processor2.py.

extract_par_section()

def multilevel_processor2.extract_par_section	(		par_contents,
			section
	)

Returns a single section (e.g. L1a, GEO, L1B, L2, etc.) from the "par" file.

Definition at line 1158 of file multilevel_processor2.py.

find_geo_file()

def multilevel_processor2.find_geo_file

(

inp_file

)

Searches for a GEO file corresponding to inp_file.  If that GEO file exists,
returns that file name; otherwise, returns None.

Definition at line 1167 of file multilevel_processor2.py.

find_geo_file2()

def multilevel_processor2.find_geo_file2	(		inp_file,
			instrument,
			lvl
	)

Searches for a GEO file corresponding to inp_file.  If that GEO file exists,
returns that file name; otherwise, returns None.

Definition at line 1182 of file multilevel_processor2.py.

find_viirs_geo_file()

def multilevel_processor2.find_viirs_geo_file	(		proc,
			first_svm_file
	)

Searches for a GEO file corresponding to first_svm_file.  If that GEO file
exists, returns that file name; otherwise, returns None.

Definition at line 1209 of file multilevel_processor2.py.

get_batch_output_name()

def multilevel_processor2.get_batch_output_name	(		file_set,
			suffix
	)

Returns the output file for a "batch" run, i.e. a process that can accept
multiple inputs, such as l2bin or l3bin.

Definition at line 1219 of file multilevel_processor2.py.

get_data_file_option()

def multilevel_processor2.get_data_file_option	(		par_contents,
			opt_text
	)

If found in par_contents, the value for the option specified by opt_text
is returned; otherwise, False is returned.

Definition at line 1257 of file multilevel_processor2.py.

get_extract_params()

def multilevel_processor2.get_extract_params

(

proc

)

Run the lonlat2pixline program and return the parameters found.

Definition at line 1268 of file multilevel_processor2.py.

get_file_date()

def multilevel_processor2.get_file_date

(

filename

)

Get a Python Date object from a recognized file name's year and day of year.

Definition at line 1303 of file multilevel_processor2.py.

get_file_handling_opts()

def multilevel_processor2.get_file_handling_opts

(

par_contents

)

Returns the values of the file handling options in par_contents.

Definition at line 1326 of file multilevel_processor2.py.

get_input_files()

def multilevel_processor2.get_input_files

(

par_data

)

Get input files found in the uber par file's ifile line, a file list file,
or both.  Ensure that the list contains no duplicates.

Definition at line 1335 of file multilevel_processor2.py.

get_input_files_type_data()

def multilevel_processor2.get_input_files_type_data

(

input_files_list

)

Returns a dictionary with the the file_type (L0, L1A, L2, etc) and
instrument for each file in the input list.

Definition at line 1362 of file multilevel_processor2.py.

get_intermediate_processors()

def multilevel_processor2.get_intermediate_processors	(		sensor,
			existing_procs,
			rules,
			lowest_source_level
	)

Create processor objects for products which are needed, but not explicitly
specified in the par file.

Definition at line 1403 of file multilevel_processor2.py.

get_intermediate_products()

def multilevel_processor2.get_intermediate_products	(		existing_prod_names,
			ruleset,
			lowest_source_level
	)

Find products which are needed, but not explicitly specified by the
par file.

Definition at line 1421 of file multilevel_processor2.py.

get_l2_extension()

def multilevel_processor2.get_l2_extension

(

)

Returns the extension for an L2 file.  For the time being, this is
just '.L2'; however, different extensions may be wanted in the future, thus
this function is in place.

Definition at line 1438 of file multilevel_processor2.py.

get_l3bin_extension()

def multilevel_processor2.get_l3bin_extension

(

)

Returns the extension for an L3 Binned file.  For the time being, this is
just '.L3bin'; however, different extensions may be wanted in the future,
thus this function is in place.

Definition at line 1446 of file multilevel_processor2.py.

get_lowest_source_level()

def multilevel_processor2.get_lowest_source_level

(

source_files

)

Find the level of the lowest level source file to be processed.

Definition at line 1454 of file multilevel_processor2.py.

get_obpg_data_file_object()

def multilevel_processor2.get_obpg_data_file_object

(

file_specification

)

Returns an obpg_data_file object for the file named in file_specification.

Definition at line 868 of file multilevel_processor2.py.

get_options()

def multilevel_processor2.get_options

(

par_data

)

Extract the options for a program to be run from the corresponding data in
the uber par file.

Definition at line 1470 of file multilevel_processor2.py.

get_output_name2()

def multilevel_processor2.get_output_name2	(		inp_files,
			targ_prog,
			suite = None,
			oformt = None,
			res = None
	)

Determine what the output name would be if targ_prog is run on input_files.

Definition at line 1487 of file multilevel_processor2.py.

get_output_name3()

def multilevel_processor2.get_output_name3	(		input_name,
			input_files,
			suffix
	)

Determine the output name for a program to be run.

Definition at line 1504 of file multilevel_processor2.py.

get_par_file_contents()

def multilevel_processor2.get_par_file_contents	(		par_file,
			acceptable_single_keys
	)

Return the contents of the input "par" file.

Definition at line 1546 of file multilevel_processor2.py.

get_processors()

def multilevel_processor2.get_processors	(		src_files,
			input_files,
			par_contents,
			files_to_delete
	)

Determine the processors which are needed.

Definition at line 1646 of file multilevel_processor2.py.

get_processors2()

def multilevel_processor2.get_processors2	(		sensor,
			par_contents,
			rules,
			lowest_source_level
	)

Determine the processors which are needed.

Definition at line 1601 of file multilevel_processor2.py.

get_required_programs()

def multilevel_processor2.get_required_programs	(		target_program,
			ruleset,
			lowest_source_level
	)

Returns the programs required too produce the desired final output.

Definition at line 1841 of file multilevel_processor2.py.

get_source_file_sets()

def multilevel_processor2.get_source_file_sets	(		proc_src_types,
			source_files,
			src_key,
			requires_all_sources
	)

Returns the set of source files needed.

Definition at line 1884 of file multilevel_processor2.py.

get_source_files()

def multilevel_processor2.get_source_files

(

input_files

)

Returns a dictionary containing the programs to be run (as keys) and the
a list of files on which that program should be run.

Definition at line 1943 of file multilevel_processor2.py.

get_source_geo_files()

def multilevel_processor2.get_source_geo_files	(		source_files,
			proc_src_types,
			proc_src_ndx
	)

:param source_files: list of source files
:param proc_src_types: list of source types for the processor
:param proc_src_ndx: index into the proc_src_types list pointing to the
                     source type to use to get the input files
:return: list of GEO files that correspond to the files in source_files

Definition at line 1864 of file multilevel_processor2.py.

get_source_products_types()

def multilevel_processor2.get_source_products_types	(		targt_prod,
			ruleset
	)

Return the list of source product types needed to produce the final product.

Definition at line 1957 of file multilevel_processor2.py.

get_traceback_message()

def multilevel_processor2.get_traceback_message

(

)

Returns an error message built from traceback data.

Definition at line 1972 of file multilevel_processor2.py.

initialze_sensors()

def multilevel_processor2.initialze_sensors

(

)

Initialize sensors.

Definition at line 1989 of file multilevel_processor2.py.

log_and_exit()

def multilevel_processor2.log_and_exit

(

error_msg

)

Record error_msg in the debug log, then exit with error_msg going to stderr
and an exit code of 1; see:
    http://docs.python.org/library/sys.html#exit.

Definition at line 2002 of file multilevel_processor2.py.

main()

def multilevel_processor2.main

(

)

main processing function.

Definition at line 2011 of file multilevel_processor2.py.

process_command_line()

def multilevel_processor2.process_command_line

(

cl_parser

)

Get arguments and options from the calling command line.
To be consistent with other OBPG programs, an underscore ('_') is used for
multiword options, instead of a dash ('-').

Definition at line 2076 of file multilevel_processor2.py.

read_file_list_file()

def multilevel_processor2.read_file_list_file

(

flf_name

)

Reads flf_name and returns the list of files to be processed.

Definition at line 2116 of file multilevel_processor2.py.

run_batch_processor()

def multilevel_processor2.run_batch_processor	(		processor,
			file_set
	)

Run a processor, e.g. l2bin, which processes batches of files.

Definition at line 2137 of file multilevel_processor2.py.

run_nonbatch_processor()

def multilevel_processor2.run_nonbatch_processor

(

processor

)

Run a processor which deals with single input files (or pairs of files in
the case of MODIS L1B processing in which GEO files are also needed).

Definition at line 2185 of file multilevel_processor2.py.

run_script()

def multilevel_processor2.run_script	(		proc,
			script_name
	)

Build the command to run the processing script which is passed in.

Definition at line 2254 of file multilevel_processor2.py.

start_logging()

def multilevel_processor2.start_logging

(

time_stamp

)

Opens log file(s) for debugging.

Definition at line 2289 of file multilevel_processor2.py.

uniqify_list()

def multilevel_processor2.uniqify_list

(

orig_list

)

Returns a list with no duplicates.  Somewhat borrowed from:
  http://www.peterbe.com/plog/uniqifiers-benchmark (example f5)

Definition at line 2312 of file multilevel_processor2.py.

Variable Documentation

cfg_data

cfg_data = None

Definition at line 2330 of file multilevel_processor2.py.

DEBUG

bool DEBUG = False

Definition at line 2327 of file multilevel_processor2.py.

FILE_USE_OPTS

list FILE_USE_OPTS = ['deletefiles', 'overwrite', 'use_existing']

Definition at line 2331 of file multilevel_processor2.py.

input_file_data

dictionary input_file_data = {}

Definition at line 2351 of file multilevel_processor2.py.

OCSSWROOT_DIR

OCSSWROOT_DIR = os.environ['OCSSWROOT']

Definition at line 2355 of file multilevel_processor2.py.

SUFFIXES

dictionary SUFFIXES

Initial value:

=  {
    'geo': 'GEO',
    'l1brsgen': 'L1B_BRS',
    'l1aextract': 'L1A.sub',
    'l1aextract_viirs': 'L1A.sub',
    'l1aextract_seawifs': 'L1A.sub',
    'l1aextract_modis': 'L1A.sub',
    'l1mapgen': 'L1B_MAP',
    'l2bin': 'L3b',
    'l2brsgen': 'L2_BRS',
    'l2extract': 'L2.sub',
    'l2gen': 'L2',
    'l2mapgen': 'L2B_MAP',
    'l3bin': 'L3b',
    'l3mapgen': 'L3m',
    'level 1a': 'L1A',
    'level 1b': 'L1B_LAC',
    'smigen': 'SMI'
}

Definition at line 2332 of file multilevel_processor2.py.