OB.DAAC Logo
NASA Logo
Ocean Color Science Software

ocssw V2022
getl0indx.c
Go to the documentation of this file.
1 /* ============================================================== */
2 /* Module: getl0indx.c */
3 /* Purpose: generate content and quality index for L0 file */
4 /* Author: B.A. Franz, General Scences Corp., 9/97 */
5 /* ============================================================== */
6 
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <string.h>
10 #include <sys/types.h>
11 #include <time.h>
12 
13 #include "swl0_proto.h"
14 
15 INT32 getl0indx(char *filename, swl0ctl *l0ctl, swl0indx *indx) {
16  swl0hdr hdr;
17  BYTE mnf[L0LEN];
18  INT16 scid[2];
19  INT16 ttag[4];
20  INT16 mnftype;
21  INT16 mnfnum;
22  FLOAT64 startSec;
23  FLOAT64 stopSec;
24  FLOAT64 utime;
25  FLOAT64 lastTime;
26  INT32 irec;
27  INT32 srec;
28  INT32 erec;
29  FILE *fp;
30  INT32 nrecs;
31  INT32 numbits;
32  INT32 numerrs;
33  INT32 totnumbits = 0;
34  INT32 totnumerrs = 0;
35  INT16 timeValid;
36 
37  /* */
38  /* Open file for reading */
39  /* */
40  if ((fp = fopen(filename, "r")) == NULL) {
41  fprintf(stderr,
42  "-E- %s line %d: unable to open %s for reading\n",
43  __FILE__, __LINE__, filename);
44  return (FATAL_ERROR);
45  }
46 
47  /* */
48  /* Read Level-0 header */
49  /* */
50  if (fread(&hdr, sizeof (hdr), 1, fp) != 1) {
51  fprintf(stderr,
52  "-E- %s line %d: error reading %s\n",
53  __FILE__, __LINE__, filename);
54  return (FATAL_ERROR);
55  }
56  if (endianess() == 1) {
57  swapc_bytes((char *) &hdr.numbits, 4, 1);
58  swapc_bytes((char *) &hdr.errbits, 4, 1);
59  swapc_bytes((char *) &hdr.startTime, 4, 1);
60  swapc_bytes((char *) &hdr.stopTime, 4, 1);
61  swapc_bytes((char *) &hdr.numlines, 4, 1);
62  }
63 
64  /* Check header magic number */
65  if (strncmp((char *) hdr.id, "CWIF", 4) != 0) {
66  fprintf(stderr,
67  "-E- %s line %d: error in header format of %s\n",
68  __FILE__, __LINE__, filename);
69  fprintf(stderr, " Header should start with CWIF string.\n");
70  fprintf(stderr, " This may not be level-0 format.\n");
71  }
72 
73  /* Invoke filetype over-ride switch, if set */
74  if (l0ctl->fileType >= 0)
75  hdr.type = l0ctl->fileType;
76 
77  /* Header AOS and LOS should be greater than 09/01/1997 */
78  if (hdr.stopTime <= hdr.startTime || hdr.startTime < 873072000) {
79  if (hdr.type != HRPT) {
80  fprintf(stderr,
81  "-E- %s line %d: error in header format of %s\n",
82  __FILE__, __LINE__, filename);
83  fprintf(stderr, " AOS and/or LOS times make no sense.\n");
84  }
85  hdr.startTime = 873072000;
86  hdr.stopTime = (INT32) time(NULL);
87  }
88 
89  /* Check validity of HRPT/GAC flag in header */
90  if (hdr.type != HRPT && hdr.type != GAC) {
91  fprintf(stderr,
92  "-E- %s line %d: error in header format of %s\n",
93  __FILE__, __LINE__, filename);
94  fprintf(stderr, " HRPT/Stored-GAC flag invalid, assuming HRPT.\n");
95  hdr.type = HRPT;
96  }
97 
98  /* */
99  /* Determine total number of frames and allocate frame quality index*/
100  /* */
101  nrecs = (filesize(filename) - sizeof (hdr)) / sizeof (mnf);
102  if (nrecs < 1) {
103  fprintf(stderr,
104  "-E- %s line %d: input file %s is empty\n",
105  __FILE__, __LINE__, filename);
106  return (FATAL_ERROR);
107  }
108  if ((indx->rec = (frameqc *) calloc(nrecs, sizeof (frameqc))) == NULL) {
109  fprintf(stderr,
110  "-E- %s line %d: error allocating memory for index array\n",
111  __FILE__, __LINE__);
112  return (FATAL_ERROR);
113  }
114 
115  /* */
116  /* Load or initialize level-0 indx structure */
117  /* */
118  strncpy(indx->l0file, filename, FILENAME_MAX);
119  indx->timeAOS = hdr.startTime;
120  indx->timeLOS = hdr.stopTime;
121  indx->type = hdr.type;
122  indx->nrecs = nrecs;
123  indx->ngac = 0;
124  indx->nlac = 0;
125  indx->nlun = 0;
126  indx->nsol = 0;
127  indx->nigc = 0;
128  indx->ntdi = 0;
129  indx->nund = 0;
130  indx->timeError = 0;
131  indx->scidError = 0;
132  indx->sohError = 0;
133  indx->bitRateRep = ((double) hdr.errbits) / MAX(hdr.numbits, 1);
134  srec = 0;
135  erec = nrecs - 1;
136 
137 
138  /* */
139  /* Get bounding times. For HRPT, all frame timetags should be */
140  /* between AOS and LOS times. GAC/LAC should be between LOS - 1 day */
141  /* and LOS time. */
142  /* */
143  if (hdr.type == HRPT) {
144  startSec = hdr.startTime;
145  stopSec = hdr.stopTime;
146  } else {
147  startSec = hdr.startTime - 86400.;
148  stopSec = hdr.stopTime + 60.;
149  }
150 
151 
152  /* */
153  /* Read through end of input file, load frame index */
154  /* */
155  printf("\nChecking telemetry for general corruption.\n");
156  for (irec = 0; irec < nrecs; irec++) {
157 
158  if (fread(mnf, L0LEN, 1, fp) != 1) {
159  fprintf(stderr,
160  "-E- %s line %d: error reading %s at rec %d\n",
161  __FILE__, __LINE__, indx->l0file, irec);
162  return (FATAL_ERROR);
163  }
164 
165  /* */
166  /* Initialize error flags for this frame */
167  /* */
168  indx->rec[irec].bitError = 0;
169  indx->rec[irec].scidError = 0;
170  indx->rec[irec].tRanError = 0;
171  indx->rec[irec].tSeqError = 0;
172  indx->rec[irec].tDifError = 0;
173  indx->rec[irec].sgaError = 0;
174  indx->rec[irec].sacError = 0;
175  indx->rec[irec].saaError = 0;
176 
177  /* */
178  /* Get frame type, number, and time */
179  /* */
180  memcpy(scid, &mnf[O_SCID], sizeof (scid));
181  memcpy(ttag, &mnf[O_TIME], sizeof (ttag));
182  if (endianess() == 1) {
183  swapc_bytes((char *) scid, 2, 2);
184  swapc_bytes((char *) ttag, 2, 4);
185  }
186  mnfnum = scid2mnfnum(scid);
187  mnftype = scid2mnftype(scid);
188  utime = ttag2unix(ttag);
189 
190  indx->rec[irec].time = utime;
191  indx->rec[irec].mnfnum = mnfnum;
192  indx->rec[irec].mnftype = mnftype;
193 
194  /* Check for Spacecraft ID field error */
195  if (scid[0] != 403 && scid[0] != 147 && scid[0] != 275) {
196  indx->rec[irec].scidError = 1;
197  printf("Invalid S/C ID, %d, at frame %d.\n", scid[0], irec);
198  }
199  if (hdr.type == HRPT && mnftype != 0) {
200  indx->rec[irec].scidError = 1;
201  printf("Invalid frame type for HRPT at frame %d.\n", irec);
202  }
203  if (hdr.type == GAC && ((mnftype > 4 && mnftype < 15) || mnftype > 15)) {
204  indx->rec[irec].scidError = 1;
205  printf("Invalid frame type for GAC/LAC at frame %d.\n", irec);
206  }
207 
208  /* */
209  /* Check for bit errors */
210  /* */
211  indx->rec[irec].bitError = bitError(mnf, &numbits, &numerrs);
212  indx->rec[irec].numBits = numbits;
213  indx->rec[irec].errBits = numerrs;
214  totnumbits += numbits;
215  totnumerrs += numerrs;
216  if (mnf[2] > 0)
217  indx->rec[irec].bitError = 1;
218  if (indx->rec[irec].bitError == 1)
219  printf("Frame %5d, FF bit errors: %d, L0 bit errors: %d\n",
220  irec, mnf[2], numerrs);
221  if (indx->rec[irec].errBits > l0ctl->maxBitErrors) {
222  printf("Max number of bit errors exceeded for this frame.\n");
223  indx->rec[irec].maxErrBits = 1;
224  }
225 
226  /* */
227  /* Check for errors in SOH headers */
228  /* */
229  if (mnfnum == 1) {
230  if (sohHdrError(&mnf[O_SGA]))
231  indx->rec[irec].sgaError = 1;
232  if (sohHdrError(&mnf[O_SAC]))
233  indx->rec[irec].sacError = 1;
234  if (sohHdrError(&mnf[O_SAA]))
235  indx->rec[irec].saaError = 1;
236  }
237 
238  /* */
239  /* Compute pixel-to-pixel variance */
240  /* */
241  indx->rec[irec].pixVariance = pixVariance(mnf);
242  }
243 
244  /* */
245  /* If this is HRPT data, the scene should be defined by contiguous */
246  /* frames with a standard time increment, so we can redefine the */
247  /* start and end times based on timetag continuity. */
248  /* */
249  if (hdr.type == HRPT) {
250 
251  printf("\nComputing AOS and LOS from frame timetags.\n");
252 
253  /* Find valid start time */
254  timeValid = 0;
255  irec = 0;
256  while (irec < nrecs && !(timeValid = timeContiguous(indx, irec)))
257  irec++;
258 
259  if (timeValid == 1) {
260  srec = MAX(irec - 1, 0);
261  startSec = indx->rec[srec].time;
262  indx->timeAOS = startSec;
263  printf("Contiguous timetags at frame %d\n", srec);
264  for (irec = srec - 1; irec >= 0; irec--)
265  indx->rec[irec].tRanError = 1;
266  } else {
267  fprintf(stderr,
268  "-E- %s line %d: no contiguous timetags found in %s\n",
269  __FILE__, __LINE__, filename);
270  exit(FATAL_ERROR);
271  }
272 
273 
274  /* Find valid stop time */
275  timeValid = 0;
276  irec = nrecs - 1;
277  while ((irec > 0 &&
278  (!(timeValid = timeContiguous(indx, irec)) ||
279  indx->rec[irec].time > startSec + l0ctl->stopTimeDelta)) ||
280  indx->rec[irec].time < startSec)
281  irec--;
282 
283  if (timeValid) {
284  erec = MIN(irec + 1, nrecs - 1);
285  stopSec = indx->rec[erec].time;
286  indx->timeLOS = stopSec;
287  printf("Contiguous timetags at frame %d\n", erec);
288  for (irec = erec + 1; irec < nrecs; irec++)
289  indx->rec[irec].tRanError = 1;
290  } else {
291  fprintf(stderr,
292  "-E- %s line %d: no contiguous timetags found in %s\n",
293  __FILE__, __LINE__, filename);
294  exit(FATAL_ERROR);
295  }
296  }
297  indx->srec = srec;
298  indx->erec = erec;
299 
300 
301  /* */
302  /* If this is GAC data, the scene should be defined by contiguous */
303  /* GAC frames at the start. We can use that knowledge to reset the */
304  /* time limits for GAC, but we only do that on request of the user. */
305  /* */
306  if (hdr.type == GAC && l0ctl->timerangeFactor > 0) {
307 
308  printf("\nComputing valid timerange from frame timetags.\n\n");
309 
310  /* Find valid start time */
311  timeValid = 0;
312  irec = 0;
313  while (irec < nrecs && (timeValid = timeContiguous(indx, irec)) == 0)
314  irec++;
315  if (timeValid == 1) {
316  startSec = indx->rec[irec].time;
317  printf("Contiguous timetags at frame %d\n", irec);
318  } else {
319  fprintf(stderr,
320  "-E- %s line %d: no contiguous timetags found in %s\n",
321  __FILE__, __LINE__, filename);
322  exit(FATAL_ERROR);
323  }
324 
325  stopSec = startSec + l0ctl->timerangeFactor * 86400.0;
326  startSec = startSec - l0ctl->timerangeFactor * 86400.0;
327  indx->timeAOS = startSec;
328  indx->timeLOS = stopSec;
329  }
330 
331 
332  /* */
333  /* Now, loop through index and Set frame quality flags */
334  /* */
335  printf("\nChecking timetag differences, limits, and sequence.\n");
336  for (irec = srec; irec <= erec; irec++) {
337 
338  mnfnum = indx->rec[irec].mnfnum;
339  mnftype = indx->rec[irec].mnftype;
340 
341  /* Time range check */
342  if (indx->rec[irec].time < startSec ||
343  indx->rec[irec].time > stopSec) {
344  printf("Time Range Error at Frame %d\n", irec);
345  indx->rec[irec].tRanError = 1;
346  }
347 
348  /* Time difference check */
349  if (!timeConsistent(indx, irec)) {
350  indx->rec[irec].tDifError = 1;
351  }
352 
353  /* Time sequence check */
354  indx->rec[irec].tSeqError = timeSeqError(indx, irec);
355  if (indx->rec[irec].tSeqError == 1)
356  printf("Time Sequence Error at Frame %d\n", irec);
357 
358  /* Time shift check */
359  if (mnftype == GACTYPE)
360  indx->rec[irec].tShfError =
361  timeShifted(indx, irec, &(indx->rec[irec].timeShift));
362  else {
363  indx->rec[irec].tShfError = 0;
364  indx->rec[irec].timeShift = 0.0;
365  }
366 
367  /* */
368  /* Update accumulation statistics */
369  /* */
370  switch (mnftype) {
371  case LACTYPE: indx->nlac += 1;
372  break;
373  case LUNTYPE: indx->nlun += 1;
374  break;
375  case SOLTYPE: indx->nsol += 1;
376  break;
377  case IGCTYPE: indx->nigc += 1;
378  break;
379  case TDITYPE: indx->ntdi += 1;
380  break;
381  case GACTYPE: indx->ngac += 1;
382  break;
383  default: indx->nund += 1;
384  break;
385  }
386  indx->scidError += indx->rec[irec].scidError;
387  indx->sohError += indx->rec[irec].sgaError ||
388  indx->rec[irec].sacError ||
389  indx->rec[irec].saaError;
390  indx->timeError += indx->rec[irec].tRanError ||
391  indx->rec[irec].tSeqError ||
392  indx->rec[irec].tDifError;
393  }
394 
395  /* */
396  /* Compute total bit errors from level-0 data */
397  /* */
398  indx->bitRateComp = (float) totnumerrs / (float) totnumbits;
399 
400  /* */
401  /* Now, make another pass to repeat the time sequence test. This */
402  /* test uses only unflagged timetags, so it may catch additional */
403  /* problems on a second pass. In addition, we impose a strict */
404  /* requirement that HRPT frames be monotonically increasing. */
405  /* */
406  printf("\nPerforming pass 2 time sequence test.\n");
407  lastTime = startSec;
408  for (irec = srec; irec <= erec; irec++) {
409 
410  if (!timeError(indx, irec)) {
411  /* Time sequence check */
412  indx->rec[irec].tSeqError = timeSeqError(indx, irec);
413  if (indx->rec[irec].tSeqError == 1) {
414  printf("Time Sequence Error at Frame %d\n", irec);
415  indx->rec[irec].tSeqError = 1;
416  indx->timeError++;
417  }
418  }
419 
420  if (indx->type == HRPT && !timeError(indx, irec)) {
421  /* Time increasing check */
422  if (indx->rec[irec].time < lastTime) {
423  printf("Decreasing timetag at frame %d\n", irec);
424  indx->rec[irec].tSeqError = 1;
425  indx->timeError++;
426  } else
427  lastTime = indx->rec[irec].time;
428  }
429  }
430 
431  fclose(fp);
432 
433  indx->timeFirst = (double) indx->timeLOS;
434  indx->timeLast = (double) indx->timeAOS;
435 
436  for (irec = srec; irec <= erec; irec++) {
437  if (!timeError(indx, irec)) {
438  indx->timeFirst = MIN(indx->rec[irec].time, indx->timeFirst);
439  indx->timeLast = MAX(indx->rec[irec].time, indx->timeLast);
440  }
441  }
442 
443  return (nrecs);
444 }
INT32 getl0indx(char *filename, swl0ctl *l0ctl, swl0indx *indx)
Definition: getl0indx.c:15
#define O_SGA
Definition: swl0_parms.h:35
BYTE timeError(swl0indx *indx, INT32 irec)
Definition: swl0_utils.c:109
#define MAX(A, B)
Definition: swl0_utils.h:26
#define MIN(x, y)
Definition: rice.h:169
#define SOLTYPE
Definition: swl0_parms.h:22
double FLOAT64
Definition: elements.h:8
#define INT32
Definition: l1_imgscale.c:3
#define NULL
Definition: decode_rs.h:63
#define O_TIME
Definition: swl0_parms.h:33
int32_t INT32
Definition: elements.h:6
#define LACTYPE
Definition: swl0_parms.h:20
#define O_SAC
Definition: swl0_parms.h:36
INT16 scid2mnftype(INT16 scid[])
Definition: swl0_utils.c:20
BYTE timeContiguous(swl0indx *indx, INT32 irec)
Definition: swl0_utils.c:178
#define GAC
Definition: l1stat.h:33
BYTE timeSeqError(swl0indx *indx, INT32 irec)
Definition: swl0_utils.c:121
int endianess(void)
determine endianess
Definition: endianess.c:10
#define IGCTYPE
Definition: swl0_parms.h:23
int swapc_bytes(char *in, int nbyte, int ntime)
Definition: swapc_bytes.c:4
#define LUNTYPE
Definition: swl0_parms.h:21
INT32 pixVariance(BYTE mnf[])
Definition: swl0_utils.c:582
unsigned char BYTE
Definition: elements.h:4
short int INT16
Definition: elements.h:5
#define HRPT
Definition: l1stat.h:35
#define FATAL_ERROR
Definition: swl0_parms.h:5
INT16 scid2mnfnum(INT16 scid[])
Definition: swl0_utils.c:29
BYTE sohHdrError(BYTE hdr[])
Definition: swl0_utils.c:416
char filename[FILENAME_MAX]
Definition: atrem_corl1.h:122
#define O_SAA
Definition: swl0_parms.h:37
integer, parameter double
#define GACTYPE
Definition: swl0_parms.h:19
BYTE timeShifted(swl0indx *indx, INT32 irec, FLOAT64 *shiftval)
Definition: swl0_utils.c:334
BYTE timeConsistent(swl0indx *indx, INT32 irec)
Definition: swl0_utils.c:255
this program makes no use of any feature of the SDP Toolkit that could generate such a then geolocation is calculated at that and then aggregated up to Resolved feature request Bug by adding three new int8 SDSs for each high resolution offsets between the high resolution geolocation and a bi linear interpolation extrapolation of the positions This can be used to reconstruct the high resolution geolocation Resolved Bug by delaying cumulation of gflags until after validation of derived products Resolved Bug by setting Latitude and Longitude to the correct fill resolving to support Near Real Time because they may be unnecessary if use of entrained ephemeris and attitude data is turned resolving bug report Corrected to filter out Aqua attitude records with missing status helping resolve bug MOD_PR03 will still correctly write scan and pixel data that does not depend upon the start time
Definition: HISTORY.txt:248
#define L0LEN
Definition: swl0_parms.h:8
PARAM_TYPE_NONE Default value No parameter is buried in the product name name_prefix is case insensitive string compared to the product name PARAM_TYPE_VIS_WAVE The visible wavelength bands from the sensor are buried in the product name The product name is compared by appending and name_suffix ie aph_412_giop where prod_ix will be set to PARAM_TYPE_IR_WAVE same search method as PARAM_TYPE_VIS_WAVE except only wavelength above are looped through but prod_ix is still based ie aph_2_giop for the second and prod_ix set to PARAM_TYPE_INT name_prefix is compared with the beginning of the product name If name_suffix is not empty the it must match the end of the product name The characters right after the prefix are read as an integer and prod_ix is set to that number strncpy(l2prod->name_prefix, "myprod", UNITLEN)
BYTE bitError(BYTE mnf[], INT32 *totbits, INT32 *toterrs)
Definition: swl0_utils.c:441
#define TDITYPE
Definition: swl0_parms.h:24
#define O_SCID
Definition: swl0_parms.h:32
INT32 filesize(const char *filename)
Definition: swl0_utils.c:93
FLOAT64 ttag2unix(INT16 ttag[])
Definition: swl0_utils.c:38