A sphere with three ocean waves in differing shades of blue. Next to the sphere, there is the bolded text "Ocean Color". Under that, there are two acronyms, separated by a vertical pipe: OB.DAAC (Ocean Biology Distributed Active Archive Center) and OBPG (Ocean Biology Processing Group). Lorem ipsum dolor sit amet
NASA Logo
Ocean Color Science Software

ocssw V2022
main_vcalmerge.c
Go to the documentation of this file.
1 /* ========================================================================
2  * vcalmerge - reads original and vicarious data from L2 file or files
3  * and writes valid cross-calibration data into an HDF/netCDF4 file
4  *
5  * Synopsis:
6  *
7  * vcalmerge par=parfile
8  *
9  * Description:
10  *
11  * Modification history:
12  *
13  * Programmer Organization Date Description of change
14  * -------------- ------------ -------- ---------------------
15  * Ewa Kwiatkowska SAIC 27 August 2003 Original development
16  * Ewa Kwiatkowska SAIC 6 November 2006 getl1rec change
17  * Ewa Kwiatkowska SAIC 9 October 2007 complete redevelopment
18  * to extract xcal data
19  * from L2 files
20  *
21  * Joel Gales Futuretech 24 October 2012 Fix sensor attribute bug
22  * Joel Gales Futuretech 14 June 2013 Add support for NETCDF4
23  *
24  * Sean Bailey Futuretech 23 July 2014 Modified to use the new
25  * calfile_utils and to use
26  * the clo library
27  * ======================================================================== */
28 
29 #include <stdio.h>
30 #include <sys/types.h>
31 #include <sys/stat.h>
32 #include <unistd.h>
33 #include <stdlib.h>
34 #include <fcntl.h>
35 #include <string.h>
36 #include <libgen.h>
37 #include <math.h>
38 
39 #include <readL2scan.h>
40 
41 #include "l12_proto.h"
42 #include <setupflags.h>
43 #include "calfile_utils.h"
44 #include <hdf5.h>
45 
46 int16_t fileID = 0;
47 #define NOVALIDPIX 111
48 
49 float ***aots, ***nlws; //arrays to contain the values needed to avg over for inversion
50 
51 /* -------------------------------------------------------------------- *
52  * main *
53  * -------------------------------------------------------------------- */
54 int main(int argc, char *argv[]) {
55 
56  int32_t iscan = 0; /* input scan number */
57  int32_t oscan = 0; /* output scan number */
58  int32_t spix = 0; /* start pixel for subscene process */
59  int32_t epix = -1; /* end pixel for subscene process */
60  int32_t ipix;
61  int32_t dpix;
62  int32_t sscan = 0; /* start scan for subscene process */
63  int32_t escan = -1; /* end scan for subscene process */
64  int32_t dscan;
65  int32_t opix = 0; /* output pixels number */
66  int32_t npix = 0; /* Number of output pixels per scan */
67  int32_t nscan = 0; /* Number of output scans */
68  int32_t npixs = 0;
69  int32_t i, j, l, np, Ltinx, vLtinx;
70  int status;
71  static int opened = 0;
72 
73  l2_prod l2_str; /* input data for L2 calibrator */
74  filehandle l2file; /* input metadata for L2 calibrator */
75  int32_t nbands, nprods;
76  int8 *iptr;
77  idDS ds_id;
78 
79  uint32_t flagusemask;
80  uint32_t required_mask;
81 
82  calstr** calrecArray; /* calrec for each pixel in a scan */
83  static int sensorID;
84  int nvars1d = 2;
85  char vars1Dnames[nvars1d][32]; // floating point one dimensional data to include in output - lon, lat
86 
87  char l2_flags[1000];
88 
89  if (argc == 1) {
90  l2gen_usage("vcalmerge");
91  return 0;
92  }
93 
94  want_verbose = 0;
95 
96  setvbuf(stdout, NULL, _IOLBF, 0);
97  setvbuf(stderr, NULL, _IOLBF, 0);
98 
99  /* Initialize file handles */
100  // cdata_();
101  filehandle_init(&l2file);
102  msl12_input_init();
103 
104  /* Parse input parameters */
105  if (msl12_input(argc, argv, "vcalmerge", &l2file) != 0) {
106  printf("-E- %s: Error parsing input parameters.\n", argv[0]);
107  exit(EXIT_FAILURE);
108  }
109 
110  if (input->ifile[0][0] == '\0') {
111  printf("-E- %s: No L2 file provided, exiting...\n", argv[0]);
112  exit(EXIT_FAILURE);
113  }
114 
115  if (access(input->ifile[0], F_OK) || access(input->ifile[0], R_OK)) {
116  printf("-E- %s: Input file '%s' does not exist or cannot open.\n",
117  argv[0], input->ifile[0]);
118  exit(EXIT_FAILURE);
119  }
120 
121  printf("\n");
122 
123  /*
124  * If output exists, check sensorID
125  *
126  */
127  int existingSensorID;
128  sensorID = l2file.sensorID;
129 
130  /* Save old HDF5 error handler */
131  H5E_auto_t old_func;
132  void *old_client_data;
133  H5Eget_auto(H5E_DEFAULT, &old_func, &old_client_data);
134 
135  /* Turn off error handling */
136  H5Eset_auto(H5E_DEFAULT, NULL, NULL);
137 
138  if (Hishdf(input->ofile[0]) == TRUE || H5Fis_hdf5(input->ofile[0]) == TRUE) {
139  ds_id = openDS(input->ofile[0]);
140  status = readAttr(ds_id, "sensorID", &existingSensorID);
141  if (status) {
142  printf("-E- %s: Problem reading output file: %s\n", argv[0], input->ofile[0]);
143  exit(EXIT_FAILURE);
144  }
145  endDS(ds_id);
146  if (sensorID != existingSensorID) {
147  printf("-E- %s: Mixing L2 data from different sensors, %s and %s.\n",
148  argv[0], sensorId2SensorName(sensorID), sensorId2SensorName(existingSensorID));
149  exit(EXIT_FAILURE);
150  }
151 
152  }
153 
154  /* Restore previous HDF5 error handler */
155  H5Eset_auto(H5E_DEFAULT, old_func, old_client_data);
156  l2_str.nrec = 0;
157  if (openL2(input->ifile[0], 0x0, &l2_str)) {
158  printf("-E- %s: Error reading L2 data %s.\n", argv[0], input->ifile[0]);
159  exit(EXIT_FAILURE);
160  }
161 
162 
163  /*
164  * Check for required data sets: mside, detnum, pixnum
165  */
166  if (l2_str.mside == NULL) {
167  printf("-E- %s: mside is missing from the L2 data file %s.\n", argv[0],
168  l2_str.filename);
169  exit(EXIT_FAILURE);
170  }
171  if (l2_str.detnum == NULL) {
172  printf("-E- %s: detnum is missing from the L2 data file %s.\n", argv[0],
173  l2_str.filename);
174  exit(EXIT_FAILURE);
175  }
176  if (l2_str.pixnum == NULL) {
177  printf("-E- %s: pixnum is missing from the L2 data file %s.\n", argv[0],
178  l2_str.filename);
179  exit(EXIT_FAILURE);
180  }
181 
182  /*
183  * Make sure all L2 products required for the xcalibration are present
184  */
185 
186  Ltinx = -1;
187  vLtinx = -1;
188  for (j = 0; j < l2_str.nprod; j++) {
189  if (strncmp(l2_str.prodname[j], "l2_flags", 8) == 0)
190  continue;
191  if (strncmp(l2_str.prodname[j], "Lt_", 3) == 0) {
192  if (Ltinx < 0)
193  Ltinx = j;
194  }
195  if (strncmp(l2_str.prodname[j], "vLt_", 4) == 0) {
196  if (vLtinx < 0)
197  vLtinx = j;
198  }
199  }
200  if (Ltinx == -1) {
201  printf(
202  "-E- %s: Lt TOA calibration data are missing from the L2 data file %s.\n",
203  argv[0], l2_str.filename);
204  exit(EXIT_FAILURE);
205  }
206  if (vLtinx == -1) {
207  printf(
208  "-E- %s: vLt TOA calibration data are missing from the L2 data file %s.\n",
209  argv[0], l2_str.filename);
210  exit(EXIT_FAILURE);
211  }
212 
213  nbands = rdsensorinfo(sensorID, 0, NULL, NULL);
214  nprods = l2_str.nprod;
215 
216  strcpy(vars1Dnames[0], "lon");
217  strcpy(vars1Dnames[1], "lat");
218 
219  /*
220  * Set up the l2_flag masking
221  */
222  strcpy(l2_flags, l2_str.flagnames);
223  l = strlen(l2_flags);
224  setupflags(l2_flags, input->flaguse, &flagusemask, &required_mask, &status,l2_str.l2_bits);
225  if (status < 0) {
226  printf("-E- %s: Error setting up L2 flags.\n", argv[0]);
227  exit(EXIT_FAILURE);
228  }
229  printf("\n");
230 
231  /*
232  * Open the output file. It will be created if necessary
233  */
234 
235  long runCounter = 0;
236  char outprod[L1_MAXPROD][32];
237  for (i = 0; i < nprods; i++) {
238  strcpy(outprod[i], l2_str.prodname[i]);
239  }
240 
241  printf("\n%s: Appending file: %s.\n", argv[0], input->ifile[0]);
242 
243  /* Set the end pixel if it was not set by command argument */
244  if (l1_input->epixl == -1 || l1_input->epixl > l2_str.nsamp)
245  l1_input->epixl = l2_str.nsamp;
246  if (l1_input->eline == -1 || l1_input->eline > l2_str.nrec)
247  l1_input->eline = l2_str.nrec;
248  if (l1_input->spixl < 1)
249  l1_input->spixl = 1;
250  if (l1_input->sline < 1)
251  l1_input->sline = 1;
252 
253  spix = MAX(l1_input->spixl - 1, 0);
254  epix = MIN(l1_input->epixl - 1, l2_str.nsamp - 1);
255  dpix = MAX(l1_input->dpixl, 1);
256  sscan = MAX(l1_input->sline - 1, 0);
257  escan = MIN(l1_input->eline - 1, l2_str.nrec - 1);
258  dscan = MAX(l1_input->dline, 1);
259 
260  if (sscan > escan || spix > epix) {
261  printf("-E- %s: scan and pixel limits make no sense.\n", argv[0]);
262  printf(" start scan = %d\n", sscan + 1);
263  printf(" end scan = %d\n", escan + 1);
264  printf(" start pixel = %d\n", spix + 1);
265  printf(" end pixel = %d\n", epix + 1);
266  exit(EXIT_FAILURE);
267  }
268 
269  npix = (epix - spix) / dpix + 1;
270  nscan = (escan - sscan) / dscan + 1;
271 
272  int quarterscans = floor(nscan / 4);
273 
274  if ((iptr = (int8 *) malloc(nscan * sizeof (int8))) == NULL) {
275  printf("-E- %s: Error allocating memory to the pixel index.\n",
276  argv[0]);
277  exit(EXIT_FAILURE);
278  }
279 
280  npixs = 0;
281  oscan = 0;
282  for (iscan = sscan; iscan <= escan; iscan += dscan) {
283 
284  if (readL2(&l2_str, l2_str.fileindex, iscan, vLtinx, NULL)) {
285  printf("%s: Error reading L2 data file %s at scan %d.\n", argv[0],
286  l2_str.filename, iscan);
287  continue;
288  }
289  if ((iscan % quarterscans) == 0)
290  printf("Selecting valid pixels for scan %d\n", iscan);
291 
292  opix = 0;
293  l = 0;
294  for (ipix = spix; ipix <= epix; ipix += dpix) {
295 
296  if (((l2_str.l2_flags[ipix] & flagusemask) == 0)
297  && (l2_str.l2_data[vLtinx][ipix] > 0.0)) {
298  l = 1;
299  ++npixs;
300  }
301  ++opix;
302  }
303 
304  if (opix != npix) {
305  printf(
306  "%s: Error: Incorrect number of pixels %d obtained in the scan %d.\n",
307  argv[0], opix, oscan);
308  }
309 
310  if (l)
311  iptr[oscan] = 1;
312  else
313  iptr[oscan] = 0;
314  ++oscan;
315  }
316 
317  printf("Number of valid cross-calibration pixels %d in file %s\n", npixs,
318  input->ifile[0]);
319 
320  /*
321  * if any valid pixels were found, write them to the output file
322  */
323  if (npixs > 0) {
324  opened = 1;
325  ds_id = calfile_open(input->ofile[0], sensorID, npixs, 1, nprods, nvars1d, outprod,
326  vars1Dnames, &runCounter, CROSSCAL);
327  /*
328  * Allocate calrec structure
329  */
330  if ((calrecArray = (calstr **) malloc(npixs * sizeof (calstr *)))
331  == NULL) {
332  printf(
333  "-E- : Error allocating memory for crosscal record structures\n");
334  exit(EXIT_FAILURE);
335  }
336  for (j = 0; j < npixs; j++) {
337  calrecArray[j] = alloc_calrec(1, nbands, nprods, nvars1d);
338  }
339 
340  oscan = 0;
341  for (iscan = sscan; iscan <= escan; iscan += dscan) {
342  np = 0;
343  /*
344  * Give a little progress...
345  */
346  if ((iscan % quarterscans) == 0)
347  printf("Storing pixels at scan %d\n", iscan);
348 
349  if (iptr[oscan++]) {
350  if (readL2(&l2_str, l2_str.fileindex, iscan, -1, NULL)) {
351  printf("%s: Error reading L2 data file %s at scan %d.\n",
352  argv[0], l2_str.filename, iscan);
353  continue;
354  }
355 
356  opix = 0;
357  for (ipix = spix; ipix <= epix; ipix += dpix) {
358 
359  if (((l2_str.l2_flags[ipix] & flagusemask) == 0)
360  && (l2_str.l2_data[Ltinx][ipix] > 0.0)
361  && (l2_str.l2_data[Ltinx][ipix] < 10000.)
362  && (l2_str.l2_data[vLtinx][ipix] > 0.0)) {
363 
364  calrecArray[np]->sensorID = sensorID;
365  calrecArray[np]->year = (int16) l2_str.year;
366  calrecArray[np]->day = (int16) l2_str.day;
367  calrecArray[np]->msec = (int32) l2_str.msec;
368  calrecArray[np]->iscan = (int16) iscan;
369  calrecArray[np]->mside = (int32) (l2_str.mside[iscan]);
370  calrecArray[np]->detnum = (int32) (l2_str.detnum[iscan]);
371  calrecArray[np]->pixnum = (int32) (l2_str.pixnum[ipix]);
372  calrecArray[np]->vars1D[0] = l2_str.longitude[ipix];
373  calrecArray[np]->vars1D[1] = l2_str.latitude[ipix];
374  for (j = 0; j < nbands; j++) {
375  calrecArray[np]->Lt[j][0] =
376  (float) l2_str.l2_data[Ltinx + j][ipix];
377  calrecArray[np]->vLt[j][0] =
378  (float) l2_str.l2_data[vLtinx + j][ipix];
379  }
380  // The calrec entries for the other request products are populated here
381  for (j = 0; j < nprods; j++) {
382  if (strcmp(l2_str.prodname[j], "pixnum") != 0
383  && strcmp(l2_str.prodname[j], "mside") != 0
384  && strncmp(l2_str.prodname[j], "Lt", 2) != 0
385  && strncmp(l2_str.prodname[j], "vLt", 3) != 0
386  && strcmp(l2_str.prodname[j], "detnum") != 0
387  && strcmp(l2_str.prodname[j], "l2_flags") != 0) {
388 
389  // fill up calrec data
390  calrecArray[np]->data[j][0] = l2_str.l2_data[j][ipix];
391  }
392  }
393 
394  ++np;
395  }
396 
397  ++opix;
398  }
399 
400  if (opix != npix) {
401  /*
402  * this really should never happen...
403  */
404  printf(
405  "%s: Error: Incorrect number of pixels %d obtained in the scan %d.\n",
406  argv[0], opix, oscan);
407  }
408  if (np > 0) {
409  // Write out the valid pixels for this scan
410 
411  for (j = 0; j < np; j++) {
412  calfile_write(ds_id, calrecArray[j], runCounter, 1, 1,
413  nprods, nbands, nvars1d, outprod, vars1Dnames, CROSSCAL);
414  runCounter++;
415  }
416  }
417  }
418  }
419 
420  // Free up some memory
421  for (i = 0; i < np; i++) {
422  free_calrec(calrecArray[i], nbands, nprods);
423  }
424  }
425 
426  free(iptr);
427 
428  if (opened == 1) {
429  if (calfile_close(ds_id) != 0) {
430  printf("Trouble closing file %s\n", input->ofile[0]);
431  exit(EXIT_FAILURE);
432  }
433  printf("Completed processing file %s\n", input->ifile[0]);
434  exit(SUCCESS);
435  } else {
436  printf("Completed processing file %s, but no valid pixels were found\n", input->ifile[0]);
437  exit(NOVALIDPIX);
438  }
439 
440 }
441 
L1_MAXPROD
#define L1_MAXPROD
Definition: filehandle.h:20
setupflags
void setupflags(char *flagdef, char *flaguse, uint32_t *flagusemask, uint32_t *required, int *status, int32_t *BITS)
Definition: setupflags.c:7
MAX
#define MAX(A, B)
Definition: swl0_utils.h:25
cubeio::int16
integer, parameter int16
Definition: cubeio.f90:3
MIN
#define MIN(x, y)
Definition: rice.h:169
SUCCESS
#define SUCCESS
Definition: ObpgReadGrid.h:15
j
int j
Definition: decode_rs.h:73
msl12_input
int msl12_input(int argc, char *argv[], const char *progName, filehandle *l1file)
Definition: msl12_input.c:4359
status
int status
Definition: l1_czcs_hdf.c:32
aots
float *** aots
Definition: main_vcalmerge.c:49
NOVALIDPIX
#define NOVALIDPIX
Definition: main_vcalmerge.c:47
openDS
idDS openDS(const char *filename)
Definition: wrapper.c:606
calfile_write
int calfile_write(idDS ds_id, calstr *calrec, int recnum, int ydim, int xdim, int nprods, int nbands, int nvars1d, char l2prods[L1_MAXPROD][32], char vars1Dnames[L1_MAXPROD][32], caltype ctype)
Definition: calfile_utils.c:348
NULL
#define NULL
Definition: decode_rs.h:63
filehandle_init
void filehandle_init(filehandle *file)
Definition: filehandle_init.c:43
TRUE
#define TRUE
Definition: rice.h:165
msl12_input_init
void msl12_input_init()
Definition: msl12_input.c:495
nscan
int32 nscan
Definition: l1_czcs_hdf.c:19
readL2
int32_t readL2(l2_prod *l2_str, int32_t ifile, int32_t recnum, int32_t iprod, unsigned char *scan_in_rowgroup)
Definition: readL2scan.c:1336
l2gen_usage
int l2gen_usage(const char *prog)
Definition: msl12_input.c:4414
input
instr * input
Definition: main_l2binmatch.cpp:27
readAttr
int readAttr(idDS ds_id, const char *nam, void *data)
l12_proto.h
CROSSCAL
@ CROSSCAL
Definition: calfile_utils.h:21
fileID
int16_t fileID
Definition: main_vcalmerge.c:46
openL2
int32_t openL2(const char *fname, const char *plist, l2_prod *l2_str)
Definition: readL2scan.c:316
oci_hdr_strip.argc
argc
Definition: oci_hdr_strip.py:7
free_calrec
void free_calrec(calstr *calrec, int nbands, int nprods)
Definition: calfile_utils.c:578
l1_input
l1_input_t * l1_input
Definition: l1_options.c:9
want_verbose
int want_verbose
Definition: genutils_globals.c:3
nlws
float *** nlws
Definition: main_vcalmerge.c:49
main
int main(int argc, char *argv[])
Definition: main_vcalmerge.c:54
alloc_calrec
calstr * alloc_calrec(int ydim, int nbands, int nprods, int nvar1d)
Definition: calfile_utils.c:516
dpix
int32 dpix
Definition: l1_czcs_hdf.c:22
nbands
int32_t nbands
Definition: atrem_corl1v3.h:190
sensorId2SensorName
const char * sensorId2SensorName(int sensorId)
Definition: sensorInfo.c:273
spix
int32 spix
Definition: l1_czcs_hdf.c:21
iscan
int32_t iscan
Definition: l1_hmodis_hdf.c:745
idDS
Definition: dfutils.h:28
epix
int32 epix
Definition: l1_czcs_hdf.c:23
rdsensorinfo
int32_t rdsensorinfo(int32_t, int32_t, const char *, void **)
Definition: rdsensorinfo.c:69
calfile_close
int calfile_close(idDS ds_id)
Definition: calfile_utils.c:210
endDS
int endDS(idDS ds_id)
Definition: wrapper.c:624
calfile_open
idDS calfile_open(char *ofile, int sensorID, int ydim, int xdim, int nprods, int nvars1d, char l2prods[L1_MAXPROD][32], char vars1Dnames[L1_MAXPROD][32], long *numExistingRecords, caltype ctype)
Definition: calfile_utils.c:48
i
int i
Definition: decode_rs.h:71
l2file
int32 l2file(int32 sdfid, int32 *nsamp, int32 *nscans, char *dtype)
Definition: l2stat_chk.c:313
strcpy
How many dimensions is the output array Default is Not sure if anything above will work correctly strcpy(l2prod->title, "no title yet")
sensorID
int32_t sensorID[MAXNFILES]
Definition: l2bin.cpp:91
npix
int npix
Definition: get_cmp.c:28
calfile_utils.h


ABOUT

Our Team

Contact Us

DATA

Get Started

Find Data

Use Data

TOOLS

SeaBASS

SeaDAS

Giovanni Data Visualizer

Worldview Mapping Interface

Mission Quality Monitor

HELP

How to Use Earthdata Search

Level 3 & 4 Browser Help

File Search Help

Search & Download Help

Earthdata Forum

Join Mailing List

How to Cite Data

CONNECT WITH US
COMPLIANCE

Web Privacy Policy

Data & Information Policy

Communications Policy

Accessibility

Freedom of Information Act

USA.gov

Responsible NASA Official: Sean Bailey
Curator: OceanColor Webmaster