OB.DAAC Logo
NASA Logo
Ocean Color Science Software

ocssw V2022
DDataset.hpp
Go to the documentation of this file.
1 /***************************************************************************
2  * NAME: DDataset.hpp
3  *
4  * DESCRIPTION: Template header file defining data structure, and basic
5  * read and write functions
6  *
7  * Created on: July, 2020
8  * Author: Sam Anderson
9 
10  ***************************************************************************/
11 
12 #ifndef INCLUDE_DDATASET_H_
13 #define INCLUDE_DDATASET_H_
14 
15 #include <string>
16 #include <vector>
17 #include <netcdf>
18 #include <boost/array.hpp>
19 #include <boost/multi_array.hpp>
20 #include <boost/lexical_cast.hpp>
21 
22 using namespace std;
23 using namespace netCDF;
24 
25 constexpr float DFILL_FLOAT = -32767.0;
26 constexpr float DFILL_TEST = -999;
27 constexpr short DFILL_SHORT = -32767;
28 constexpr unsigned char DFILL_UBYTE = 255;
29 constexpr short MAX_SHORT = 32765;
30 
31 enum class dtype
32 {
33  BYTE = NC_BYTE,
34  CHAR = NC_CHAR,
35  SHORT = NC_SHORT,
36  INT = NC_INT,
37  FLOAT = NC_FLOAT,
38  DOUBLE = NC_DOUBLE,
39  UBYTE = NC_UBYTE,
40  USHORT = NC_USHORT,
41  UINT = NC_UINT,
42  INT64 = NC_INT64,
43  UINT64 = NC_UINT64,
44  STRING = NC_STRING
45 };
46 
47 template<typename U, size_t rank>
48 boost::array<U, rank> stl2boost(const vector<U> &vec)
49 {
50  assert(vec.size() == rank);
51  boost::array<U, rank> result;
52  std::copy(vec.begin(), vec.end(), result.begin());
53  return result;
54 };
55 
56 class ddata
57 {
58 public:
59  size_t rank;
61  vector<size_t> start;
62  vector<size_t> count;
63  bool bshort;
64  const void *ptr;
65 
66  ddata(){
67  rank = 1;
69  ptr = nullptr;
70  bshort = false;
71  };
72 
73  ddata(dtype itype,
74  vector<size_t> istart,
75  vector<size_t> icount,
76  void *iptr)
77  {
78  rank = icount.size();
79  type = itype;
80  start = istart;
81  count = icount;
82  ptr = iptr;
83  bshort = false;
84  };
85 
86  virtual ~ddata(){
87  start.clear();
88  count.clear();
89  };
90 
91  void setFormat(bool bshrt) {bshort = bshrt;}
92 
93  void putAtt(NcFile* ncfile, string name){
94  switch(type) {
95  case dtype::FLOAT:
96  ncfile->putAtt(name, ncFloat, count[0], static_cast<const float*> (ptr));
97  break;
98  case dtype::INT:
99  ncfile->putAtt(name, ncInt, count[0], static_cast<const int*> (ptr));
100  break;
101  case dtype::SHORT:
102  ncfile->putAtt(name, ncShort, count[0], static_cast<const short*> (ptr));
103  break;
104  case dtype::UBYTE:
105  ncfile->putAtt(name, ncUbyte, count[0], static_cast<const unsigned char*> (ptr));
106  break;
107  case dtype::BYTE:
108  ncfile->putAtt(name, ncByte, count[0], static_cast<const char*> (ptr));
109  break;
110  case dtype::CHAR:
111  ncfile->putAtt(name, ncChar, count[0], static_cast<const char*> (ptr));
112  break;
113  case dtype::DOUBLE:
114  ncfile->putAtt(name, ncDouble, count[0], static_cast<const double*> (ptr));
115  break;
116  case dtype::USHORT:
117  ncfile->putAtt(name, ncUshort, count[0], static_cast<const unsigned short*> (ptr));
118  break;
119  case dtype::UINT:
120  ncfile->putAtt(name, ncUint, count[0], static_cast<const unsigned int*> (ptr));
121  break;
122  case dtype::INT64:
123  ncfile->putAtt(name, ncInt64, count[0], static_cast<const long long*> (ptr));
124  break;
125  case dtype::UINT64:
126  ncfile->putAtt(name, ncUint64, count[0], static_cast<const unsigned long long*> (ptr));
127  break;
128  case dtype::STRING:
129  ncfile->putAtt(name, static_cast<const char*> (ptr));
130  break;
131  }
132  }
133 
134  void putAtt(NcVar& var, string name, string sval){
135  switch(type) {
136  case dtype::FLOAT:
137  var.putAtt(name, ncFloat, boost::lexical_cast<float> (sval));
138  break;
139  case dtype::INT:
140  var.putAtt(name, ncInt, boost::lexical_cast<int> (sval));
141  break;
142  case dtype::SHORT:
143  var.putAtt(name, ncShort, boost::lexical_cast<short> (sval));
144  break;
145  case dtype::UBYTE:
146  var.putAtt(name, ncUbyte, boost::lexical_cast<unsigned int> (sval));
147  break;
148  case dtype::BYTE:
149  var.putAtt(name, ncByte, boost::lexical_cast<char> (sval));
150  break;
151  case dtype::CHAR:
152  var.putAtt(name, ncChar, boost::lexical_cast<char> (sval));
153  break;
154  case dtype::DOUBLE:
155  var.putAtt(name, ncDouble, boost::lexical_cast<double> (sval));
156  break;
157  case dtype::USHORT:
158  var.putAtt(name, ncUshort, boost::lexical_cast<unsigned short> (sval));
159  break;
160  case dtype::UINT:
161  var.putAtt(name, ncUint, boost::lexical_cast<unsigned int> (sval));
162  break;
163  case dtype::INT64:
164  var.putAtt(name, ncInt64, boost::lexical_cast<long long> (sval));
165  break;
166  case dtype::UINT64:
167  var.putAtt(name, ncUint64, boost::lexical_cast<unsigned long long> (sval));
168  break;
169  case dtype::STRING:
170  var.putAtt(name, sval);
171  break;
172  }
173  }
174 
175  void setFill(NcVar& var, string sval){
176  const string& str = sval;
177  switch(type) {
178  case dtype::FLOAT:
179  if(bshort) {
180  var.setFill(true, DFILL_SHORT);
181  } else {
182  var.setFill(true, DFILL_FLOAT);
183  }
184  break;
185  case dtype::INT:
186  var.setFill(true, boost::lexical_cast<int> (str));
187  break;
188  case dtype::SHORT:
189  var.setFill(true, boost::lexical_cast<short> (str));
190  break;
191  case dtype::UBYTE:
192  var.setFill(true, boost::lexical_cast<unsigned int> (str));
193  break;
194  case dtype::BYTE:
195  var.setFill(true, boost::lexical_cast<char> (str));
196  break;
197  case dtype::CHAR:
198  var.setFill(true, boost::lexical_cast<char> (str));
199  break;
200  case dtype::DOUBLE:
201  var.setFill(true, boost::lexical_cast<double> (str));
202  break;
203  case dtype::USHORT:
204  var.setFill(true, boost::lexical_cast<unsigned short> (str));
205  break;
206  case dtype::UINT:
207  var.setFill(true, boost::lexical_cast<unsigned int> (str));
208  break;
209  case dtype::INT64:
210  var.setFill(true, boost::lexical_cast<long long> (str));
211  break;
212  case dtype::UINT64:
213  var.setFill(true, boost::lexical_cast<unsigned long long> (str));
214  break;
215  case dtype::STRING:
216  var.setFill(true, str);
217  break;
218  }
219  }
220 
221  void putVar(NcVar& var){
222  switch(type) {
223  case dtype::FLOAT:
224  if(bshort && count.size()==2) {
225  float scale, offset;
226  std::map<std::string,NcVarAtt> matts = var.getAtts();
227  if (matts.find("scale_factor") != matts.end()) {
228  matts["scale_factor"].getValues(&scale);
229  matts["add_offset"].getValues(&offset);
230  }
231  boost::multi_array<short, 2> spts;
232  boost::array<size_t,2> bdimv = stl2boost<size_t,2>(count);
233  spts.resize(bdimv);
234  for (size_t i=0; i<spts.num_elements(); i++) {
235  if (((float*)ptr)[i] <= DFILL_TEST) {
236  ((short*)spts.origin())[i] = DFILL_SHORT;
237  } else {
238  float ftemp = (((float*)ptr)[i] - offset)/scale;
239  ((short*)spts.origin())[i] =
240  (ftemp > (float) MAX_SHORT) ? MAX_SHORT : (short) ftemp;
241  }
242  }
243  var.putVar(start, count, spts.origin());
244  } else {
245  var.putVar(start, count, static_cast<const float*> (ptr));
246  }
247  break;
248  case dtype::INT:
249  var.putVar(start, count, static_cast<const int*> (ptr));
250  break;
251  case dtype::SHORT:
252  var.putVar(start, count, static_cast<const short*> (ptr));
253  break;
254  case dtype::UBYTE:
255  var.putVar(start, count, static_cast<const unsigned char*> (ptr));
256  break;
257  case dtype::BYTE:
258  var.putVar(start, count, static_cast<const char*> (ptr));
259  break;
260  case dtype::CHAR:
261  var.putVar(start, count, static_cast<const char*> (ptr));
262  break;
263  case dtype::DOUBLE:
264  var.putVar(start, count, static_cast<const double*> (ptr));
265  break;
266  case dtype::USHORT:
267  var.putVar(start, count, static_cast<const unsigned short*> (ptr));
268  break;
269  case dtype::UINT:
270  var.putVar(start, count, static_cast<const unsigned int*> (ptr));
271  break;
272  case dtype::INT64:
273  var.putVar(start, count, static_cast<const long long*> (ptr));
274  break;
275  case dtype::UINT64:
276  var.putVar(start, count, static_cast<const unsigned long long*> (ptr));
277  break;
278  case dtype::STRING:
279  var.putVar(start, count, static_cast<const char*> (ptr));
280  break;
281  }
282  }
283 };
284 
285 class ddstr: public ddata
286 {
287 public:
288  string str;
289 
290  ddstr(){};
291  ~ddstr(){};
292 
293  ddstr(const string istr) {
294  str = istr;
296  start.push_back(0);
297  count.push_back(1);
298  ptr = str.c_str();
299  }
300 };
301 
302 template<typename T>
303 class ddval: public ddata
304 {
305 public:
306  T val;
307 
308  ddval(){};
309  ~ddval(){};
310 
311  ddval<T>(dtype itype, const T ival) {
312  val = ival;
313  type = itype;
314  start.push_back(0);
315  count.push_back(1);
316  ptr = &val;
317  }
318 };
319 
320 template<typename T, size_t ndim>
321 class ddma: public ddata
322 {
323 public:
324  boost::multi_array<T, ndim> pts;
325 
326  ddma(){};
327  ~ddma(){};
328 
330  vector<size_t> istart,
331  vector<size_t> icount) {
332  rank = ndim;
333  type = itype;
334  start = istart;
335  count = icount;
336  boost::array<size_t, ndim> bdimv = stl2boost<size_t,ndim>(count);
337  pts.resize(bdimv);
338  ptr = pts.origin();
339  }
340 
341  ddma<T,ndim>(NcGroup& ncgrp,
342  string name,
343  dtype itype,
344  vector<size_t> istart,
345  vector<size_t> icount)
346  {
347  NcVar var = ncgrp.getVar(name);
348  if (!var.isNull()) {
349  type = itype;
350  rank = var.getDimCount();
351  vector<NcDim> dims = var.getDims();
352  size_t beg = (rank == ndim+1) ? 1 : 0;
353  for (size_t i=beg; i<rank; i++){
354  start.push_back(istart[i]);
355  count.push_back(icount[i]);
356  }
357  boost::array<size_t, ndim> bdimv = stl2boost<size_t,ndim>(count);
358  pts.resize(bdimv);
359  ptr = pts.origin();
360  var.getVar(istart, icount, (T*)ptr);
361 
362  T factor = 1.0;
363  T offset = 0.0;
364  T fillvalue = 0;
365  std::map<std::string,NcVarAtt> matts = var.getAtts();
366  if (matts.find("scale_factor") != matts.end()) {
367  matts["scale_factor"].getValues(&factor);
368  matts["add_offset"].getValues(&offset);
369  }
370  matts["_FillValue"].getValues(&fillvalue);
371  for (size_t i=0; i<pts.num_elements(); i++) {
372  if (((T*)ptr)[i] == fillvalue || ((T*)ptr)[i] == fillvalue-2) {
373  ((T*)ptr)[i] = (T)DFILL_FLOAT;
374  } else {
375  ((T*)ptr)[i] = ((T*)ptr)[i]*factor + offset;
376  }
377  }
378  } else {
379  ptr = nullptr;
380  }
381  }
382 };
383 
384 #endif
constexpr short MAX_SHORT
Definition: DDataset.hpp:29
virtual ~ddata()
Definition: DDataset.hpp:86
@ SHORT
signed 2 byte integer
void putAtt(NcFile *ncfile, string name)
Definition: DDataset.hpp:93
ddval()
Definition: DDataset.hpp:308
constexpr unsigned char DFILL_UBYTE
Definition: DDataset.hpp:28
string str
Definition: DDataset.hpp:288
void putVar(NcVar &var)
Definition: DDataset.hpp:221
boost::array< U, rank > stl2boost(const vector< U > &vec)
Definition: DDataset.hpp:48
@ UBYTE
unsigned 1 byte int
dtype type
Definition: DDataset.hpp:60
constexpr float DFILL_TEST
Definition: DDataset.hpp:26
ddata()
Definition: DDataset.hpp:66
boost::multi_array< T, ndim > pts
Definition: DDataset.hpp:324
@ FLOAT
single precision floating point number
~ddstr()
Definition: DDataset.hpp:291
ddata(dtype itype, vector< size_t > istart, vector< size_t > icount, void *iptr)
Definition: DDataset.hpp:73
void setFill(NcVar &var, string sval)
Definition: DDataset.hpp:175
@ UINT64
unsigned 8-byte int
ddma()
Definition: DDataset.hpp:326
~ddval()
Definition: DDataset.hpp:309
unsigned char BYTE
Definition: elements.h:4
@ STRING
string
void putAtt(NcVar &var, string name, string sval)
Definition: DDataset.hpp:134
@ USHORT
unsigned 2-byte int
ddstr(const string istr)
Definition: DDataset.hpp:293
@ BYTE
signed 1 byte integer
constexpr float DFILL_FLOAT
Definition: DDataset.hpp:25
@ INT64
signed 8-byte int
const char * str
Definition: l1c_msi.cpp:35
~ddma()
Definition: DDataset.hpp:327
dtype
Definition: DDataset.hpp:31
@ UINT
unsigned 4-byte int
Extra metadata that will be written to the HDF4 file l2prod rank
void copy(double **aout, double **ain, int n)
void setFormat(bool bshrt)
Definition: DDataset.hpp:91
@ INT
signed 4 byte integer
vector< size_t > count
Definition: DDataset.hpp:62
ddstr()
Definition: DDataset.hpp:290
l2prod offset
bool bshort
Definition: DDataset.hpp:63
size_t rank
Definition: DDataset.hpp:59
const void * ptr
Definition: DDataset.hpp:64
vector< size_t > start
Definition: DDataset.hpp:61
int i
Definition: decode_rs.h:71
msiBandIdx val
Definition: l1c_msi.cpp:34
@ CHAR
ISO/ASCII character.
@ DOUBLE
double precision floating point number
constexpr short DFILL_SHORT
Definition: DDataset.hpp:27
int count
Definition: decode_rs.h:79