OB.DAAC Logo
NASA Logo
Ocean Color Science Software

ocssw V2022
val_extract.h
Go to the documentation of this file.
1 
5 #ifndef ___VAL_EXTRACT_H_
6 #define ___VAL_EXTRACT_H_
7 
8 #include "argpar.h"
9 #include "olog.h"
10 #include "shash.h"
11 
12 #include <netcdf.h>
13 #include <stdbool.h>
14 #include <stdint.h>
15 #include <stdlib.h>
16 #include <time.h>
17 
18 #ifndef VALEXTRACT_API_VERSION
19 
20 #define VALEXTRACT_API_VERSION 2007005
21 
22 #define VALEXTRACT_API_VERSION_STR "2.7.5"
23 #endif
24 
26 #define VALEXTRACT_ERR_NONE EXIT_SUCCESS
27 
28 #define VALEXTRACT_ERR_POINT_NOT_FOUND 99
29 
31 #define VALEXTRACT_ERR_UNKNOWN 100
32 
33 #define VALEXTRACT_ERR_NCFILE_ERR 101
34 
35 #define VALEXTRACT_ERR_NCFILE_INVALID 102
36 
37 #define VALEXTRACT_ERR_FLAG 103
38 
40 #define VALEXTRACT_ERR_VARIABLE 104
41 
42 #define VALEXTRACT_ERR_INPUT 105
43 
44 #define VALEXTRACT_ERR_L2QC 106
45 
48 
50 #define VALEXTRACT_KEY_INIT 1
51 
53 #define VALEXTRACT_KEY_FILE 2
54 
56 #define VALEXTRACT_KEY_VAR 3
57 
58 #define VALEXTRACT_KEY_SUCCESS 4
59 
60 #define VALEXTRACT_KEY_ERROR 5
61 
62 #define VALEXTRACT_KEY_FINI 6
63 
65 #define VALEXTRACT_CMD_STOP 2
66 
68 #define VALEXTRACT_UNSET -255
69 
79 typedef int (*val_extract_parser)(int key, void *nc_input, void *user_input);
80 
87 typedef struct val_extract_valid_range {
88  char *prefix;
89  double min, max;
91 
103 typedef struct val_extract_arguments {
105  const char *ifile;
106 
108  int box_size;
109 
111  double box_size_km;
112 
118  int end_line;
123 
127  double start_lat;
129  double end_lat;
131  double start_lon;
133  double end_lon;
134 
140 
142  char **products;
145 
149  char **ignore_flags;
152 
156  char **count_flags;
159 
161  char **l2qc_flags;
167 
172 
175  char **skip_stats;
178 
181 
184 
187 
190 
195  void *user_input;
197 
205 
209 
219 
222 void unflatten_index(int index, int ndims, const int *dims, int *result);
223 
225 const char *val_extract_version();
226 
229 const char *val_extract_api_version();
230 
231 /* The following should go into an nc_read library */
232 
233 typedef struct nc_point {
234  size_t line, pixel;
235 } nc_point;
236 
237 typedef struct nc_box {
238  size_t start[2], count[2];
239 } nc_box;
240 
241 typedef struct nc_file {
242  const char *file_path;
244  int ngrps, *gids;
247  uint32_t *flag_masks;
248  char *flag_string;
251 } nc_file;
252 
253 typedef struct nc_region {
260  uint32_t *pixel_flags;
261  double lat, lon;
262  time_t utime;
263  char ascii_time[24];
264 } nc_region;
265 
266 typedef struct nc_var_stats {
268  int count;
269  double min, max, mean, median, stddev, rms;
270  double center_value;
271 } nc_var_stats;
272 
273 typedef struct nc_var {
276  char *name, *group_name;
277  int gid, varid;
280  double scale, offset, fill;
281  int ndims, *dim_ids;
284  nc_type data_type;
285  size_t data_sizeof;
286  double *valid_data;
288  void *data;
289 } nc_var;
290 
291 size_t nc_get_region_size(nc_region *region);
292 size_t nc_get_region_dim_size(nc_region *region, int dim_index);
293 
294 #define declare_nc_get_varr(type, suffix) \
295  int nc_get_varr ## suffix(int ncid, int varid, nc_region *region, type *p); \
296  int nc_get_varrd ## suffix(int ncid, int varid, nc_region *region, int dim_index, type *p);
297 
298 declare_nc_get_varr(char, _text)
299 declare_nc_get_varr(unsigned char, _uchar)
300 declare_nc_get_varr(signed char, _schar)
301 declare_nc_get_varr(short, _short)
302 declare_nc_get_varr(int, _int)
303 declare_nc_get_varr(long, _long)
304 declare_nc_get_varr(float, _float)
305 declare_nc_get_varr(double, _double)
306 declare_nc_get_varr(unsigned short, _ushort)
307 declare_nc_get_varr(unsigned int, _uint)
308 declare_nc_get_varr(long long, _longlong)
309 declare_nc_get_varr(unsigned long long, _ulonglong)
310 declare_nc_get_varr(char *, _string)
311 declare_nc_get_varr(void,)
312 
313 /* The following should go into a math library */
314 
315 #define declare_compare(type, suffix) int compare ## suffix(const void *a, const void *b)
316 declare_compare(int8_t, _int8);
317 declare_compare(uint8_t, _uint8);
318 declare_compare(int16_t, _int16);
319 declare_compare(uint16_t, _uint16);
320 declare_compare(int32_t, _int32);
321 declare_compare(uint32_t, _uint32);
322 declare_compare(int64_t, _int64);
323 declare_compare(uint64_t, _uint64);
324 declare_compare(float, _float);
325 declare_compare(double, _double);
326 
327 #endif /* ___VAL_EXTRACT_H_ */
double median
Definition: val_extract.h:269
void * data
Definition: val_extract.h:288
int valid_range_count
How many ignore_flags are present.
Definition: val_extract.h:171
an array had not been initialized Several spelling and grammar corrections were which is read from the appropriate MCF the above metadata values were hard coded A problem calculating the average background DN for SWIR bands when the moon is in the space view port was corrected The new algorithm used to calculate the average background DN for all reflective bands when the moon is in the space view port is now the same as the algorithm employed by the thermal bands For non SWIR changes in the averages are typically less than Also for non SWIR the black body DNs remain a backup in case the SV DNs are not available For SWIR the changes in computed averages were larger because the old which used the black body suffered from contamination by the micron leak As a consequence of the if SV DNs are not available for the SWIR the EV pixels will not be the granule time is used to identify the appropriate tables within the set given for one LUT the first two or last two tables respectively will be used for the interpolation If there is only one LUT in the set of it will be treated as a constant LUT The manner in which Earth View data is checked for saturation was changed Previously the raw Earth View DNs and Space View DNs were checked against the lookup table values contained in the table dn_sat The change made is to check the raw Earth and Space View DNs to be sure they are less than the maximum saturation value and to check the Space View subtracted Earth View dns against a set of values contained in the new lookup table dn_sat_ev The metadata configuration and ASSOCIATEDINSTRUMENTSHORTNAME from the MOD02HKM product The same metatdata with extensions and were removed from the MOD021KM and MOD02OBC products ASSOCIATEDSENSORSHORTNAME was set to MODIS in all products These changes are reflected in new File Specification which users may consult for exact the pow functions were eliminated in Emissive_Cal and Emissive bands replaced by more efficient code Other calculations throughout the code were also made more efficient Aside from a few round off there was no difference to the product The CPU time decreased by about for a day case and for a night case A minor bug in calculating the uncertainty index for emissive bands was corrected The frame index(0-based) was previously being used the frame number(1-based) should have been used. There were only a few minor changes to the uncertainty index(maximum of 1 digit). 3. Some inefficient arrays(Sigma_RVS_norm_sq) were eliminated and some code lines in Preprocess_L1A_Data were moved into Process_OBCEng_Emiss. There were no changes to the product. Required RAM was reduced by 20 MB. Now
bool uses_control_points
Definition: val_extract.h:278
int skip_stats_count
How many skip_stats products are present.
Definition: val_extract.h:177
Master structure containing options, document strings, child parsers, and text filters....
Definition: argpar.h:398
char ** count_flags
If given, count all pixels with ANY of these flags. This is added to count_flags_mask.
Definition: val_extract.h:156
#define declare_compare(type, suffix)
Definition: val_extract.h:315
int start_line
Either center line for a box extract or the beginning line for regions.
Definition: val_extract.h:116
int * flag_counts
Definition: val_extract.h:259
time_t utime
Definition: val_extract.h:262
int count_flag_count
How many count_flags are present.
Definition: val_extract.h:158
nc_box * boxes
Definition: val_extract.h:257
#define declare_nc_get_varr(type, suffix)
Definition: val_extract.h:294
int box_size
Box width (and height) for extract, in pixels.
Definition: val_extract.h:108
bool has_offset
Definition: val_extract.h:279
int pixel_count
Definition: val_extract.h:258
int gid
Definition: val_extract.h:277
nc_point center
Definition: val_extract.h:255
int valid_data_count
Definition: val_extract.h:287
double * l2qc_thresholds
If given, use these thresholds for the L2QC process, aborting processing if any are thresholds are su...
Definition: val_extract.h:164
char ** flag_meanings
Definition: val_extract.h:249
Passed into the library function to control the processing. Many of the fields will be unspecified.
Definition: val_extract.h:103
void val_extract_clear_args(val_extract_arguments *arguments)
Clear an argument struct to make it suitable for using the library without using argpar.
Definition: val_extract.c:127
const char * val_extract_api_version()
Returns a string representation of the library's implemented API version number, without a label.
Definition: val_extract.c:2635
int * dim_lengths
Definition: val_extract.h:243
bool geospatial_only
Only process geospatial variables, those with lines and pixels as the only dimensions.
Definition: val_extract.h:136
bool initialized
Definition: val_extract.h:267
char ** ignore_flags
If given, ignore all pixels with ANY of these flags. This is added to ignore_flags_mask.
Definition: val_extract.h:149
bool has_scale
Definition: val_extract.h:279
Given to val_extract_arguments, these contain the valid ranges, inclusive, of variables....
Definition: val_extract.h:87
double optics_threshold
If given, OPTICS algorithm threshold for preventing reaching over data gaps. Lower = stricter.
Definition: val_extract.h:189
int flagged_pixel_count
Definition: val_extract.h:258
char * flag_string
Definition: val_extract.h:248
Implementation-specific, generic type to store the shash object.
Definition: shash.c:40
size_t nc_get_region_dim_size(nc_region *region, int dim_index)
Definition: val_extract.c:2683
bool is_geospatial
Definition: val_extract.h:278
int ngrps
Definition: val_extract.h:244
void * user_input
If given, this is passed to the val_extract_parser for arbitrary use by the user.
Definition: val_extract.h:195
A simple logger, capable of dispatching log events to multiple end points.
size_t data_sizeof
Definition: val_extract.h:285
int pixel_control_dimid
Definition: val_extract.h:245
double end_lon
End longitude for regions.
Definition: val_extract.h:133
int ndims
Definition: val_extract.h:281
int(* val_extract_parser)(int key, void *nc_input, void *user_input)
Pointer to a callback function to call for each argument parsed.
Definition: val_extract.h:79
int ndims
Definition: val_extract.h:243
size_t pixel
Definition: val_extract.h:234
bool geospatial_to_double
Read all geospatial variables as doubles, regardless of original type, useful for avoiding copy-and-p...
Definition: val_extract.h:139
int ignore_flag_count
How many ignore_flags are present.
Definition: val_extract.h:151
double center_value
Definition: val_extract.h:270
double start_lon
Either center longitude for a box extract or the beginning longitude for regions.
Definition: val_extract.h:131
nc_region * region
Definition: val_extract.h:275
uint32_t * flag_masks
Definition: val_extract.h:247
char * name
Definition: val_extract.h:276
nc_var_stats iqr_stats
Definition: val_extract.h:283
shash * attributes
Definition: val_extract.h:282
double stddev
Definition: val_extract.h:269
olog * log
olog object to use for messages, defaults to global logger.
Definition: val_extract.h:186
bool has_fill
Definition: val_extract.h:279
nc_var_stats filtered_stats
Definition: val_extract.h:283
double end_lat
End latitude for regions.
Definition: val_extract.h:129
int end_pixel
End pixel for regions.
Definition: val_extract.h:122
uint32_t count_flags_mask
If given, count all pixels with ANY of these flag bits sets.
Definition: val_extract.h:154
int varid
Definition: val_extract.h:277
argpar val_extract_argpar
argpar structure used for making programs that inherit options from this library.
Definition: val_extract.c:523
int line_dimid
Definition: val_extract.h:245
int control_point_size
Definition: val_extract.h:245
size_t nc_get_region_size(nc_region *region)
Definition: val_extract.c:2675
Definition: olog.c:11
bool lat_and_lon
Whether to use latitudes and longitudes instead of lines and pixels.
Definition: val_extract.h:125
int start_pixel
Either center pixel for a box extract or the beginning pixel for regions.
Definition: val_extract.h:120
uint32_t ignore_flags_mask
If given, ignore all pixels with ANY of these flag bits sets.
Definition: val_extract.h:147
double offset
Definition: val_extract.h:280
double box_size_km
Box radius for extract, in kilometers.
Definition: val_extract.h:111
bool variable_atts
Include all variables attributes in output files.
Definition: val_extract.h:183
bool line_and_pixel
Whether to use lines and pixels instead of latitudes and longitudes.
Definition: val_extract.h:114
char ** l2qc_flags
If given, use these flags for the L2QC process, paired with the l2qc_thresholds input.
Definition: val_extract.h:161
void unflatten_index(int index, int ndims, const int *dims, int *result)
Given dimension lengths from an nc_file and a one dimensional index, find the corresponding n-dimensi...
Definition: val_extract.c:2604
int pixel_dimid
Definition: val_extract.h:245
int l2qc_threshold_count
How many l2qc_thresholds are present.
Definition: val_extract.h:166
double lon
Definition: val_extract.h:261
nc_var_stats stats
Definition: val_extract.h:283
A simple dictionary library for storing strings.
size_t start[2]
Definition: val_extract.h:238
int val_extract_clean(val_extract_arguments *arguments)
Clean up stuff malloc'd by the argpar callback. Should be called at the end of processing if val_extr...
Definition: val_extract.c:2540
val_extract_parser val_extract_parser
Required, this is the function that is given the nc_region and nc_var structures after they are proce...
Definition: val_extract.h:193
int val_extract(val_extract_arguments *arguments)
Process a small section of a Level-2 NetCDF file.
Definition: val_extract.c:589
int box_count
Definition: val_extract.h:256
shash * attributes
Definition: val_extract.h:250
nc_type data_type
Definition: val_extract.h:284
const char * file_path
Definition: val_extract.h:242
const char * val_extract_version()
Returns a string representation of the library's version number, without a label.
Definition: val_extract.c:2632
Library for reading command-line arguments in the form of key=value.
int flag_count
Definition: val_extract.h:246
int * dim_ids
Definition: val_extract.h:281
size_t count[2]
Definition: val_extract.h:238
int end_line
End line for regions.
Definition: val_extract.h:118
char * group_name
Definition: val_extract.h:276
double fill
Definition: val_extract.h:280
char ascii_time[24]
Definition: val_extract.h:263
double start_lat
Either center latitude for a box extract or the beginning latitude for regions.
Definition: val_extract.h:127
int unflagged_pixel_count
Definition: val_extract.h:258
double lat
Definition: val_extract.h:261
bool global_atts
Create .global file containing all global attributes.
Definition: val_extract.h:180
char ** products
If given, only process the NetCDF variables listed.
Definition: val_extract.h:142
double scale
Definition: val_extract.h:280
nc_file * file
Definition: val_extract.h:254
char ** skip_stats
If given, override default list of products where statistics are not meaningful (such as l2_flags).
Definition: val_extract.h:175
size_t line
Definition: val_extract.h:234
int product_count
How many NetCDF variables should be processed.
Definition: val_extract.h:144
val_extract_valid_range * valid_ranges
If given, use these to filter variables' data used for statistics.
Definition: val_extract.h:169
nc_file * file
Definition: val_extract.h:274
double * valid_data
Definition: val_extract.h:286
const char * ifile
Path to Level 2 NetCDF file.
Definition: val_extract.h:105
uint32_t * pixel_flags
Definition: val_extract.h:260
int * gids
Definition: val_extract.h:244