OB.DAAC Logo
NASA Logo
Ocean Color Science Software

ocssw V2022
L3File.cpp
Go to the documentation of this file.
1 #include <L3File.h>
2 #include <genutils.h>
3 #include <boost/algorithm/string.hpp>
4 
5 #include "L3ShapeIsine.h"
6 
7 using namespace std;
8 //using namespace boost::geometry;
9 
10 namespace l3 {
11 
12 //----------------------------------------------------------------------------
13 // L3Bin
14 //----------------------------------------------------------------------------
15 
16 L3Bin::L3Bin(int32_t numProducts) {
17  internalInit(numProducts);
18 }
19 
20 L3Bin::L3Bin(const L3Bin &bin) {
21  internalCopy(bin);
22 }
23 
24 L3Bin::~L3Bin() {
25  free(sums);
26  free(sumSquares);
27 }
28 
29 void L3Bin::clear() {
30  binNum = -1;
31  recordNum = -1;
32  nobs = 0;
33  nscenes = 0;
34  timeRec = 0.0;
35  weights = 0.0;
36  quality = qualityUnused;
37  for (int i = 0; i < numProducts; i++) {
38  sums[i] = 0.0;
39  sumSquares[i] = 0.0;
40  }
41 }
42 
48 void L3Bin::internalAllocate(int32_t numProducts) {
49  if (numProducts != this->numProducts) {
50  if (sums)
51  free(sums);
52  if (sumSquares)
53  free(sumSquares);
54  sums = (float*) allocateMemory(numProducts * sizeof (float),
55  "sums for L3Bin");
56  sumSquares = (float*) allocateMemory(numProducts * sizeof (float),
57  "sumSquares for L3Bin");
58  this->numProducts = numProducts;
59  }
60 }
61 
66 void L3Bin::internalInit(int32_t numProducts) {
67  this->numProducts = 0;
68  sums = sumSquares = NULL;
69  internalAllocate(numProducts);
70  clear();
71 }
72 
78 void L3Bin::internalCopy(const L3Bin &bin) {
79  binNum = bin.binNum;
80  recordNum = bin.recordNum;
81  nobs = bin.nobs;
82  nscenes = bin.nscenes;
83  quality = bin.quality;
84  timeRec = bin.timeRec;
85  weights = bin.weights;
86 
87  internalAllocate(bin.numProducts);
88  memcpy(sums, bin.sums, numProducts * sizeof (float));
89  memcpy(sumSquares, bin.sumSquares, numProducts * sizeof (float));
90 }
91 
96 void L3Bin::internalAdd(const L3Bin &bin) {
97  if (numProducts != bin.numProducts) {
98  setNumProducts(bin.numProducts);
99  }
100  if (bin.quality != qualityUnused) {
101  if (bin.quality < quality) {
102  clear();
103  quality = bin.quality;
104  }
105  }
106  binNum = bin.binNum; // just copy the binNum
107  nobs += bin.nobs;
108  nscenes += bin.nscenes;
109  timeRec += bin.timeRec;
110  weights += bin.weights;
111  for (int i = 0; i < numProducts; i++) {
112  sums[i] += bin.sums[i];
113  sumSquares[i] += bin.sumSquares[i];
114  }
115 }
116 
117 void L3Bin::checkProdId(int32_t prodId) const {
118  if (prodId < 0) {
119  fprintf(stderr,
120  "-E- L3Bin::checkProdId - prodId (%d) must be positive\n",
121  prodId);
122  exit(EXIT_FAILURE);
123  }
124  if (prodId >= numProducts) {
125  fprintf(stderr,
126  "-E- L3Bin::checkProdId - prodId (%d) is greater than the numProducts (%d)\n",
127  prodId, numProducts);
128  exit(EXIT_FAILURE);
129  }
130 }
131 
132 L3Bin& L3Bin::operator=(const L3Bin &bin) {
133  internalCopy(bin);
134  return *this;
135 }
136 
137 L3Bin& L3Bin::operator+=(const L3Bin &bin) {
138  internalAdd(bin);
139  return *this;
140 }
141 
142 void L3Bin::addWeighted(const L3Bin &bin, float weighting) {
143  if (bin.quality != qualityUnused) {
144  if (bin.quality < quality) {
145  clear();
146  quality = bin.quality;
147  }
148  }
149  nobs += bin.nobs;
150  nscenes += bin.nscenes;
151  timeRec += bin.timeRec;
152  weights += bin.weights * weighting;
153  for (int i = 0; i < numProducts; i++) {
154  sums[i] += bin.sums[i] * weighting;
155  sumSquares[i] += bin.sumSquares[i] * weighting;
156  }
157 }
158 
163 void L3Bin::setNumProducts(int32_t numProducts) {
164  if (numProducts < 1) {
165  fprintf(stderr,
166  "-E- L3Bin::setNumProducts - numProducts (%d) must be at least 1\n",
167  numProducts);
168  exit(EXIT_FAILURE);
169  }
170  internalAllocate(numProducts);
171  clear();
172 }
173 
178 int32_t L3Bin::getNumProducts() const {
179  return numProducts;
180 }
181 
186 int64_t L3Bin::getBinNum() const {
187  return binNum;
188 }
189 
194 int32_t L3Bin::getNobs() const {
195  return nobs;
196 }
197 
202 int32_t L3Bin::getNscenes() const {
203  return nscenes;
204 }
205 
210 int64_t L3Bin::getRecordNum() const {
211  return recordNum;
212 }
213 
218 float L3Bin::getObsTime() const {
219  return timeRec / nobs;
220 }
221 
226 float L3Bin::getWeights() const {
227  return weights;
228 }
229 
234 uint8_t L3Bin::getQuality() const {
235  return quality;
236 }
237 
243 float L3Bin::getSum(int32_t prodId) const {
244  checkProdId(prodId);
245  return sums[prodId];
246 }
247 
253 float L3Bin::getSumSquares(int32_t prodId) const {
254  checkProdId(prodId);
255  return sumSquares[prodId];
256 }
257 
263 float L3Bin::getMean(int32_t prodId) const {
264  checkProdId(prodId);
265  return sums[prodId] / weights;
266 }
267 
273 float L3Bin::getVariance(int32_t prodId) const {
274  checkProdId(prodId);
275  if (weights * weights > nscenes) {
276  double val = sums[prodId] / weights;
277  val = (sumSquares[prodId] / weights) - (val * val);
278  return val * weights * weights / (weights * weights - nscenes);
279  }
280  return 0.0;
281 }
282 
288 float L3Bin::getStdev(int32_t prodId) const {
289  return sqrt(getVariance(prodId));
290 }
291 
292 //----------------------------------------------------------------------------
293 // L3Row
294 //----------------------------------------------------------------------------
295 
296 L3Row::L3Row(int32_t row, int64_t numBins, int32_t numProducts) {
297  this->row = row;
298  this->numBins = numBins;
299  this->numProducts = numProducts;
300  binArray.resize(numBins);
301  for (int64_t i = 0; i < numBins; i++) {
302  binArray[i] = new L3Bin(numProducts);
303  }
304  lastBin = numBins / 2;
305 }
306 
307 L3Row::~L3Row() {
308  setNumBins(0);
309 }
310 
311 int32_t L3Row::getRow() const {
312  return row;
313 }
314 
315 int32_t L3Row::getNumProducts() const {
316  return numProducts;
317 }
318 
319 int64_t L3Row::getNumBins() const {
320  return numBins;
321 }
322 
323 void L3Row::setNumBins(int64_t numBins) {
324  this->numBins = numBins;
325  while (binArray.size() > (uint64_t) numBins) {
326  delete binArray.back();
327  binArray.pop_back();
328  }
329  while (binArray.size() < (uint64_t) numBins) {
330  binArray.push_back(new L3Bin(numProducts));
331  }
332  lastBin = numBins / 2;
333 }
334 
335 L3Bin* L3Row::getBin(int64_t binNum) {
336  L3Bin* l3Bin;
337  int64_t hi, mid, lo;
338  int64_t tmpBinNum;
339  int64_t tmp;
340 
341  if (numBins == 0)
342  return NULL;
343 
344  // start at the bin we found last time
345  lo = 0;
346  hi = numBins - 1;
347  mid = lastBin;
348 
349  while (lo <= hi) {
350  l3Bin = binArray[mid];
351  tmpBinNum = l3Bin->getBinNum();
352  if (tmpBinNum == binNum) {
353  lastBin = mid + 1;
354  if (lastBin >= numBins)
355  lastBin--;
356  if (l3Bin->nscenes == 0)
357  return NULL;
358  else
359  return l3Bin;
360  }
361  tmp = mid + (binNum - tmpBinNum);
362  if (binNum > tmpBinNum) {
363  lo = mid + 1;
364  if (hi > tmp)
365  hi = tmp;
366  } else {
367  hi = mid - 1;
368  if (lo < tmp)
369  lo = tmp;
370  }
371  mid = (lo + hi) / 2;
372  }
373  return NULL;
374 }
375 
381 L3Bin* L3Row::getBinByIndex(int32_t index) {
382  if (index < 0 || index >= numBins)
383  return NULL;
384  return binArray[index];
385 }
386 
387 //----------------------------------------------------------------------------
388 // L3File
389 //----------------------------------------------------------------------------
390 
391 L3File::L3File() {
392  shape = NULL;
393  binObj = NULL;
394  numCacheRows = 500;
395  baseRecord = NULL;
396  extentbin = NULL;
397  sumBuffer = NULL;
398  qualityBuffer = NULL;
399  prodMap = NULL;
400  numProds = 0;
401 }
402 
403 L3File::~L3File() {
404  close();
405  if (shape)
406  delete shape;
407  if (binObj)
408  delete binObj;
409 }
410 
411 void L3File::clearCache() {
412  L3Row* l3Row;
413  while (!rowList.empty()) {
414  l3Row = rowList.back();
415  delete l3Row;
416  rowList.pop_back();
417  }
418 }
419 
420 void L3File::setNumCacheRows(int32_t numRows) {
421  numCacheRows = numRows;
422  L3Row* l3Row;
423  while (rowList.size() > (uint32_t) numRows) {
424  l3Row = rowList.back();
425  delete l3Row;
426  rowList.pop_back();
427  }
428 }
429 
430 bool L3File::open(const char* fileName) {
431  close();
432  int tmpWantVerbose = want_verbose;
433  want_verbose = 0;
434  binObj = Hdf::openBinObject(fileName);
435  want_verbose = tmpWantVerbose;
436  if (binObj == NULL)
437  return false;
438  shape = new L3ShapeIsine(binObj->nrows);
439  initRecordLookup();
440  return true;
441 }
442 
443 void L3File::close() {
444  if (shape) {
445  delete shape;
446  shape = NULL;
447  }
448  if (binObj) {
449  delete binObj;
450  binObj = NULL;
451  }
452  if (sumBuffer) {
453  free(sumBuffer);
454  sumBuffer = NULL;
455  }
456  if (qualityBuffer) {
457  free(qualityBuffer);
458  qualityBuffer = NULL;
459  }
460  if (baseRecord) {
461  free(baseRecord);
462  baseRecord = NULL;
463  }
464  if (extentbin) {
465  free(extentbin);
466  extentbin = NULL;
467  }
468  clearCache();
469  if (prodMap) {
470  free(prodMap);
471  prodMap = NULL;
472  }
473 }
474 
475 meta_l3bType* L3File::getMetaData() {
476  if (binObj) {
477  return &binObj->meta_l3b;
478  }
479  return NULL;
480 }
481 
486 int L3File::initRecordLookup() {
487 
488  if (binObj->nrows == 0) {
489  fprintf(stderr,
490  "-E- %s %d: initRecordLookup: Number of rows needs to be set and bin grid must be initialized first.\n",
491  __FILE__, __LINE__);
492  exit(EXIT_FAILURE);
493  }
494 
495  // remove memory if it exists
496  if (baseRecord) {
497  free(baseRecord);
498  }
499  baseRecord = (int64_t*) allocateMemory(binObj->nrows * sizeof (int64_t),
500  "baseRecord");
501  if (extentbin) {
502  free(extentbin);
503  }
504  extentbin = (int32_t*) allocateMemory(binObj->nrows * sizeof (int32_t),
505  "extbin");
506 
507  int64_t index = 0;
508  for (int row = 0; row < binObj->nrows; row++) {
509  binObj->readBinIndex(row);
510  int32_t extent = binObj->get_ext();
511  extentbin[row] = extent;
512  if (extent > 0) {
513  baseRecord[row] = index;
514  index += extent;
515  } else {
516  baseRecord[row] = -1;
517  }
518  }
519  return 0;
520 }
521 
529 int64_t L3File::rowbin2record(int32_t row, int64_t bin) {
530  int64_t recordIndex;
531 
532  recordIndex = baseRecord[row];
533  if (recordIndex == -1) // there are no records in this row
534  return -1;
535  L3Row* l3Row = getRow(row);
536  if (l3Row == NULL || l3Row->getNumBins() == 0)
537  return -1;
538  L3Bin* l3Bin = l3Row->getBin(bin);
539  if (l3Bin == NULL)
540  return -1;
541  return l3Bin->getRecordNum();
542 }
543 
550 int64_t L3File::rowcol2record(int32_t row, int32_t col) {
551  int64_t bin;
552  bin = shape->rowcol2bin(row, col);
553  return rowbin2record(row, bin);
554 }
555 
562 int64_t L3File::latlon2record(float lat, float lon) {
563  int32_t row, col;
564  int64_t bin;
565  shape->latlon2rowcol(lat, lon, row, col);
566  bin = shape->rowcol2bin(row, col);
567  return rowbin2record(row, bin);
568 }
569 
575 int64_t L3File::bin2record(int64_t bin) {
576  int32_t row;
577  row = shape->bin2row(bin);
578  return rowbin2record(row, bin);
579 }
580 
581 int32_t L3File::getNumProducts() {
582  if (binObj == NULL)
583  return 0;
584  return binObj->nprod();
585 }
586 
588  return binObj->getProdName(index);
589 }
590 
591 bool L3File::setActiveProductList(const char* prodStr) {
592  //free the sum buffer since it depends on the number of products
593  if (sumBuffer) {
594  free(sumBuffer);
595  sumBuffer = NULL;
596  }
597  clearCache();
598  int status = binObj->read((char*) prodStr);
599  if (status == -1)
600  return false;
601 
602  // find number of products
603  string tmpStr(prodStr);
604  activeProdNameList.clear();
605  boost::split(activeProdNameList, tmpStr, boost::is_any_of(","));
606 
607  // populate the product map
608  prodMap = (size_t*) allocateMemory(
609  activeProdNameList.size() * sizeof (size_t), "prodMap");
610  numProds = activeProdNameList.size();
611  for (size_t myIndex = 0; myIndex < numProds; myIndex++) {
612  boost::trim(activeProdNameList[myIndex]);
613  for (int32_t objIndex = 0; objIndex < binObj->n_active_prod;
614  objIndex++) {
615  if (activeProdNameList[myIndex].compare(
616  binObj->getActiveProdName(objIndex)) == 0) {
617  prodMap[myIndex] = objIndex;
618  break;
619  }
620  }
621  }
622 
623  outBin.setNumProducts(activeProdNameList.size());
624 
625  return true;
626 }
627 
628 int32_t L3File::getNumActiveProducts() {
629  return activeProdNameList.size();
630 }
631 
632 string L3File::getActiveProductName(size_t index) {
633  return activeProdNameList[index];
634 }
635 
636 L3Row* L3File::readRow(int32_t row) {
637  L3Row* l3Row;
638 
639  if (row < 0 || row >= binObj->nrows)
640  return NULL;
641 
642  int32_t extent = extentbin[row];
643  int64_t base = baseRecord[row];
644  if (rowList.size() < (uint32_t) numCacheRows) {
645  l3Row = new L3Row(row, extent, numProds);
646  } else {
647  l3Row = rowList.back();
648  rowList.pop_back();
649  l3Row->setNumBins(extent);
650  l3Row->row = row;
651  }
652  rowList.push_front(l3Row);
653 
654  if (extent == 0)
655  return l3Row;
656 
657  // read the bin list
658  binObj->readBinList(extent, base);
659 
660  // allocate the sum buffer if necessary
661  if (sumBuffer == NULL)
662  sumBuffer = (float*) allocateMemory(
663  4 * binObj->nrows * binObj->n_active_prod * sizeof (float),
664  "L3File::sumBuffer");
665 
666  binObj->setDataPtrAbsolute(base);
667  binObj->readSums(sumBuffer, extent, -1);
668 
669  if (qualityBuffer)
670  binObj->readQual((uint8_t*) qualityBuffer, extent, base);
671 
672  L3Bin* l3Bin;
673  size_t sumOffset;
674  for (int i = 0; i < extent; i++) {
675  l3Bin = l3Row->binArray[i];
676  l3Bin->binNum = binObj->get_bin_num(i);
677  l3Bin->recordNum = base + i;
678  l3Bin->nobs = binObj->get_nobs(i);
679  l3Bin->nscenes = binObj->get_nscenes(i);
680  l3Bin->timeRec = binObj->get_time_rec(i);
681  l3Bin->weights = binObj->get_weights(i);
682  for (size_t prod = 0; prod < numProds; prod++) {
683  sumOffset = prodMap[prod] * extent * 2 + i * 2;
684  l3Bin->sums[prod] = sumBuffer[sumOffset];
685  l3Bin->sumSquares[prod] = sumBuffer[sumOffset + 1];
686  }
687  if (qualityBuffer)
688  l3Bin->quality = qualityBuffer[i];
689  else
690  l3Bin->quality = L3Bin::qualityUnused;
691  }
692  return l3Row;
693 }
694 
695 L3Row* L3File::getRow(int32_t row) {
696  L3Row* l3Row;
697 
698  // check first cache position
699  list<L3Row*>::iterator it = rowList.begin();
700  if (it != rowList.end()) {
701  l3Row = *it;
702  if (l3Row->getRow() == row)
703  return l3Row;
704  }
705 
706  // check the rest of the cache
707  for (it++; it != rowList.end(); it++) {
708  l3Row = *it;
709  if (l3Row->getRow() == row) {
710  if (it != rowList.begin()) {
711  rowList.erase(it);
712  rowList.push_front(l3Row);
713  }
714  return l3Row;
715  }
716  }
717 
718  // not found in cache, so read the row
719  return readRow(row);
720 }
721 
726 int32_t L3File::getNumRows() {
727  if (shape) {
728  return shape->getNumRows();
729  }
730  return -1;
731 }
732 
739 L3Bin* L3File::getClosestBin(float lat, float lon) {
740  if (shape == NULL) {
741  fprintf(stderr,
742  "-E- %s %d: need to open L3File before looking for a bin.\n",
743  __FILE__, __LINE__);
744  exit(EXIT_FAILURE);
745  }
746  int32_t row, col;
747  shape->latlon2rowcol(lat, lon, row, col);
748  L3Row* l3Row = getRow(row);
749  if (l3Row == NULL)
750  return NULL;
751  int64_t binNum = shape->rowcol2bin(row, col);
752  return l3Row->getBin(binNum);
753 }
754 
761 L3Bin* L3File::getBin(int32_t row, int32_t col) {
762  if (shape == NULL) {
763  fprintf(stderr,
764  "-E- %s %d: need to open L3File before looking for a bin.\n",
765  __FILE__, __LINE__);
766  exit(EXIT_FAILURE);
767  }
768  if (row < 0 || row >= shape->getNumRows())
769  return NULL;
770  if (col < 0 || col >= shape->getNumCols(row))
771  return NULL;
772 
773  L3Row* l3Row = getRow(row);
774  if (l3Row == NULL)
775  return NULL;
776  int64_t binNum = shape->rowcol2bin(row, col);
777  return l3Row->getBin(binNum);
778 }
779 
784 Hdf::hdf_bin* L3File::getHdfBinObject() const {
785  return binObj;
786 }
787 
788 bool L3File::hasQuality() {
789  if (binObj) {
790  return binObj->has_qual();
791  }
792  return false;
793 }
794 
795 void L3File::setQualityProcessing(bool val) {
796  if (qualityBuffer)
797  free(qualityBuffer);
798  qualityBuffer = NULL;
799  if (val) {
800  if (shape == NULL) {
801  fprintf(stderr,
802  "-E- %s %d: need to open L3File before setting qualityProcessing.\n",
803  __FILE__, __LINE__);
804  exit(EXIT_FAILURE);
805  }
806  qualityBuffer = (uint8_t*) allocateMemory(2 * shape->getNumRows(),
807  "L3File::qualityBuffer");
808  }
809 }
810 
811 bool L3File::getQualityProcessing() const {
812  if (qualityBuffer)
813  return true;
814  else
815  return false;
816 }
817 
818 } // namespace l3
an array had not been initialized Several spelling and grammar corrections were which is read from the appropriate MCF the above metadata values were hard coded A problem calculating the average background DN for SWIR bands when the moon is in the space view port was corrected The new algorithm used to calculate the average background DN for all reflective bands when the moon is in the space view port is now the same as the algorithm employed by the thermal bands For non SWIR changes in the averages are typically less than Also for non SWIR the black body DNs remain a backup in case the SV DNs are not available For SWIR the changes in computed averages were larger because the old which used the black body suffered from contamination by the micron leak As a consequence of the if SV DNs are not available for the SWIR the EV pixels will not be the granule time is used to identify the appropriate tables within the set given for one LUT the first two or last two tables respectively will be used for the interpolation If there is only one LUT in the set of it will be treated as a constant LUT The manner in which Earth View data is checked for saturation was changed Previously the raw Earth View DNs and Space View DNs were checked against the lookup table values contained in the table dn_sat The change made is to check the raw Earth and Space View DNs to be sure they are less than the maximum saturation value and to check the Space View subtracted Earth View dns against a set of values contained in the new lookup table dn_sat_ev The metadata configuration and ASSOCIATEDINSTRUMENTSHORTNAME from the MOD02HKM product The same metatdata with extensions and were removed from the MOD021KM and MOD02OBC products ASSOCIATEDSENSORSHORTNAME was set to MODIS in all products These changes are reflected in new File Specification which users may consult for exact the pow functions were eliminated in Emissive_Cal and Emissive bands replaced by more efficient code Other calculations throughout the code were also made more efficient Aside from a few round off there was no difference to the product The CPU time decreased by about for a day case and for a night case A minor bug in calculating the uncertainty index for emissive bands was corrected The frame index(0-based) was previously being used the frame number(1-based) should have been used. There were only a few minor changes to the uncertainty index(maximum of 1 digit). 3. Some inefficient arrays(Sigma_RVS_norm_sq) were eliminated and some code lines in Preprocess_L1A_Data were moved into Process_OBCEng_Emiss. There were no changes to the product. Required RAM was reduced by 20 MB. Now
virtual bool has_qual()=0
int status
Definition: l1_czcs_hdf.c:32
float * sumSquares
Definition: L3File.h:74
void * allocateMemory(size_t numBytes, const char *name)
Definition: allocateMemory.c:7
#define NULL
Definition: decode_rs.h:63
L3Bin * getBin(int64_t binNum)
Definition: L3File.cpp:335
int32_t nscenes
Definition: L3File.h:69
hdf_bin * openBinObject(const char *binFileName)
Definition: bin_io.cpp:32
std::vector< L3Bin * > binArray
Definition: L3File.h:125
float32 base
Definition: maplists.h:106
int32_t row
Definition: L3File.h:122
float * lat
int32_t nobs
Definition: L3File.h:68
int32_t nobs
Definition: atrem_cor.h:93
data_t tmp
Definition: decode_rs.h:74
uint8_t quality
Definition: L3File.h:70
float weights
Definition: L3File.h:72
int32_t getRow() const
Definition: L3File.cpp:311
int want_verbose
int64_t getNumBins() const
Definition: L3File.cpp:319
int64_t getBinNum() const
Definition: L3File.cpp:186
string & trim(string &s, const string &delimiters)
Definition: EnvsatUtil.cpp:29
int64_t binNum
Definition: L3File.h:66
void getProductName(char *name, l2prodstr *product)
float timeRec
Definition: L3File.h:71
void setNumBins(int64_t numBins)
Definition: L3File.cpp:323
int64_t getRecordNum() const
Definition: L3File.cpp:210
int16_t * nscenes
Definition: l2bin.cpp:86
float * lon
Definition: L3File.cpp:10
int64_t recordNum
Definition: L3File.h:67
float * sums
Definition: L3File.h:73
int i
Definition: decode_rs.h:71
msiBandIdx val
Definition: l1c_msi.cpp:34