bes  Updated for version 3.20.10
HDF5CFArray.cc
Go to the documentation of this file.
1 // This file is part of the hdf5_handler implementing for the CF-compliant
2 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3 //
4 // This is free software; you can redistribute it and/or modify it under the
5 // terms of the GNU Lesser General Public License as published by the Free
6 // Software Foundation; either version 2.1 of the License, or (at your
7 // option) any later version.
8 //
9 // This software is distributed in the hope that it will be useful, but
10 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12 // License for more details.
13 //
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 //
18 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
20 // Suite 203, Champaign, IL 61820
21 
30 
31 #include "config_hdf5.h"
32 #include <iostream>
33 #include <sstream>
34 #include <cassert>
35 #include <BESDebug.h>
36 #include <sys/stat.h>
37 #include <libdap/InternalErr.h>
38 
39 #include <libdap/Str.h>
40 #include "HDF5RequestHandler.h"
41 #include "HDF5CFArray.h"
42 #include "h5cfdaputil.h"
43 #include "ObjMemCache.h"
44 
45 using namespace std;
46 using namespace libdap;
47 
48 
49 BaseType *HDF5CFArray::ptr_duplicate()
50 {
51  return new HDF5CFArray(*this);
52 }
53 
54 // Read in an HDF5 Array
55 bool HDF5CFArray::read()
56 {
57 
58  BESDEBUG("h5","Coming to HDF5CFArray read "<<endl);
59  if(length() == 0)
60  return true;
61 
62  if((NULL == HDF5RequestHandler::get_lrdata_mem_cache()) &&
63  NULL == HDF5RequestHandler::get_srdata_mem_cache()){
64  read_data_NOT_from_mem_cache(false,NULL);
65  return true;
66  }
67 
68  // Flag to check if using large raw data cache or small raw data cache.
69  short use_cache_flag = 0;
70 
71  // The small data cache is checked first to reduce the resources to operate the big data cache.
72  if(HDF5RequestHandler::get_srdata_mem_cache() != NULL) {
73  if(((cvtype == CV_EXIST) && (islatlon != true)) || (cvtype == CV_NONLATLON_MISS)
74  || (cvtype == CV_FILLINDEX) ||(cvtype == CV_MODIFY) ||(cvtype == CV_SPECIAL)){
75 
76  if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==true)
77  use_cache_flag = 1;
78  }
79  }
80 
81  // If this varible doesn't fit the small data cache, let's check if it fits the large data cache.
82  if(use_cache_flag !=1) {
83 
84  if(HDF5RequestHandler::get_lrdata_mem_cache() != NULL) {
85 
86  // This is the trival case.
87  // If no information is provided in the configuration file of large data cache,
88  // just cache the lat/lon varible per file.
89  if(HDF5RequestHandler::get_common_cache_dirs() == false) {
90  if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
91  || (cvtype == CV_EXIST && islatlon == true)) {
92 #if 0
93 //cerr<<"coming to use_cache_flag =2 "<<endl;
94 #endif
95  // Only the data with the numeric datatype DAP2 and CF support are cached.
96  if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==true)
97  use_cache_flag = 2;
98  }
99  }
100  else {// Have large data cache configuration info.
101 
102  // Need to check if we don't want to cache some CVs, now
103  // this only applies to lat/lon CV.
104  if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
105  || (cvtype == CV_EXIST && islatlon == true)) {
106 
107  vector<string> cur_lrd_non_cache_dir_list;
108  HDF5RequestHandler::get_lrd_non_cache_dir_list(cur_lrd_non_cache_dir_list);
109 
110  // Check if this file is included in the non-cache directory
111  if( (cur_lrd_non_cache_dir_list.size() == 0) ||
112  ("" == check_str_sect_in_list(cur_lrd_non_cache_dir_list,filename,'/'))) {
113 
114  // Only data with the numeric datatype DAP2 and CF support are cached.
115  if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==true)
116  use_cache_flag = 3;
117  }
118  }
119  // Here we allow all the variable names to be cached.
120  // The file path that includes the variables can also included.
121  vector<string> cur_lrd_var_cache_file_list;
122  HDF5RequestHandler::get_lrd_var_cache_file_list(cur_lrd_var_cache_file_list);
123  if(cur_lrd_var_cache_file_list.size() >0){
124 #if 0
126 //cerr<<"lrd var cache is "<<cur_lrd_var_cache_file_list[i]<<endl;
127 #endif
128  if(true == check_var_cache_files(cur_lrd_var_cache_file_list,filename,varname)){
129 #if 0
130 //cerr<<"varname is "<<varname <<endl;
131 //cerr<<"have var cached "<<endl;
132 #endif
133 
134  // Only the data with the numeric datatype DAP2 and CF support are cached.
135  if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==true)
136  use_cache_flag = 4;
137  }
138  }
139  }
140  }
141  }
142 
143  if(0 == use_cache_flag)
144  read_data_NOT_from_mem_cache(false,NULL);
145  else {// memory cache cases
146 
147  string cache_key;
148 
149  // Possibly we have common lat/lon dirs,so check here.
150  if( 3 == use_cache_flag){
151  vector<string> cur_cache_dlist;
152  HDF5RequestHandler::get_lrd_cache_dir_list(cur_cache_dlist);
153  string cache_dir = check_str_sect_in_list(cur_cache_dlist,filename,'/');
154  if(cache_dir != "")
155  cache_key = cache_dir + varname;
156  else {
157  cache_key = filename + varname;
158  // If this lat/lon is not in the common dir. list, it is still cached as a general lat/lon.
159  // Change the flag to 2.
160  use_cache_flag = 2;
161  }
162 
163  }
164  else
165  cache_key = filename + varname;
166 
167  handle_data_with_mem_cache(dtype,total_elems,use_cache_flag,cache_key,is_dap4);
168 
169  }
170 
171  return true;
172 }
173 
174 // Reading data not from memory cache: The data can be read from the disk cache or can be read via the HDF5 APIs
175 void HDF5CFArray::read_data_NOT_from_mem_cache(bool add_mem_cache,void*buf) {
176 
177  vector<int>offset;
178  vector<int>count;
179  vector<int>step;
180  vector<hsize_t> hoffset;
181  vector<hsize_t>hcount;
182  vector<hsize_t>hstep;
183  int nelms = 1;
184 
185  if (rank <= 0)
186  throw InternalErr (__FILE__, __LINE__,
187  "The number of dimension of the variable is <=0 for an array.");
188  else {
189 
190  offset.resize(rank);
191  count.resize(rank);
192  step.resize(rank);
193  hoffset.resize(rank);
194  hcount.resize(rank);
195  hstep.resize(rank);
196  nelms = format_constraint (&offset[0], &step[0], &count[0]);
197  for (int i = 0; i <rank; i++) {
198  hoffset[i] = (hsize_t) offset[i];
199  hcount[i] = (hsize_t) count[i];
200  hstep[i] = (hsize_t) step[i];
201  }
202  }
203 
204  hid_t dsetid = -1;
205  hid_t dspace = -1;
206  hid_t mspace = -1;
207  hid_t dtypeid = -1;
208  hid_t memtype = -1;
209 
210  bool data_from_disk_cache = false;
211  bool data_to_disk_cache = false;
212 
213  // Check if the disk cache can be applied.
214  bool use_disk_cache = valid_disk_cache();
215 
216  string cache_fpath;
217 
218  if(true == use_disk_cache) {
219 
220  BESDEBUG("h5","Coming to use disk cache "<<endl);
221 
222  unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
223  string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
224  string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
225 
226  string cache_fname=HDF5CFUtil::obtain_cache_fname(diskcache_prefix,filename,varname);
227  cache_fpath = diskcache_dir + "/"+ cache_fname;
228 
229  int temp_total_elems = 1;
230  for (unsigned int i = 0; i <dimsizes.size();i++)
231  temp_total_elems = temp_total_elems*dimsizes[i];
232  short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
233  // CHECK: I think when signed 8-bit needs to be converted to int16, dtype_size should also change.
234  if(is_dap4 == false && dtype==H5CHAR)
235  dtype_size = 2;
236 
237  int expected_file_size = dtype_size *temp_total_elems;
238  int fd = 0;
239  HDF5DiskCache *disk_cache = HDF5DiskCache::get_instance(disk_cache_size,diskcache_dir,diskcache_prefix);
240  if( true == disk_cache->get_data_from_cache(cache_fpath, expected_file_size,fd)) {
241 
242  vector<size_t> offset_size_t;
243  offset_size_t.resize(rank);
244  for(int i = 0; i <rank;i++)
245  offset_size_t[i] = (size_t)offset[i];
246  size_t offset_1st = INDEX_nD_TO_1D(dimsizes,offset_size_t);
247  vector<size_t>end;
248  end.resize(rank);
249  for (int i = 0; i < rank; i++)
250  end[i] = offset[i] +(count[i]-1)*step[i];
251  size_t offset_last = INDEX_nD_TO_1D(dimsizes,end);
252 #if 0
253 //cerr<<"offset_1d is "<<offset_1st <<endl;
254 //cerr<<"offset_last is "<<offset_last <<endl;
255 #endif
256  size_t total_read = dtype_size*(offset_last-offset_1st+1);
257 
258  off_t fpos = lseek(fd,dtype_size*offset_1st,SEEK_SET);
259  if (-1 == fpos) {
260  disk_cache->unlock_and_close(cache_fpath);
261  disk_cache->purge_file(cache_fpath);
262  }
263 
265  else
266  data_from_disk_cache = obtain_cached_data(disk_cache,cache_fpath,fd, step,count,total_read,dtype_size);
267 
268  }
269 
270  if(true == data_from_disk_cache)
271  return;
272  else
273  data_to_disk_cache = true;
274 
275  }
276 
277 // END CACHE
278 
279  bool pass_fileid = HDF5RequestHandler::get_pass_fileid();
280  if(false == pass_fileid) {
281  if ((fileid = H5Fopen(filename.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT))<0) {
282  ostringstream eherr;
283  eherr << "HDF5 File " << filename
284  << " cannot be opened. "<<endl;
285  throw InternalErr (__FILE__, __LINE__, eherr.str ());
286  }
287  }
288 
289  if ((dsetid = H5Dopen(fileid,varname.c_str(),H5P_DEFAULT))<0) {
290  HDF5CFUtil::close_fileid(fileid,pass_fileid);
291  ostringstream eherr;
292  eherr << "HDF5 dataset " << varname
293  << " cannot be opened. "<<endl;
294  throw InternalErr (__FILE__, __LINE__, eherr.str ());
295  }
296 
297  if ((dspace = H5Dget_space(dsetid))<0) {
298 
299  H5Dclose(dsetid);
300  HDF5CFUtil::close_fileid(fileid,pass_fileid);
301  ostringstream eherr;
302  eherr << "Space id of the HDF5 dataset " << varname
303  << " cannot be obtained. "<<endl;
304  throw InternalErr (__FILE__, __LINE__, eherr.str ());
305  }
306 
307  if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
308  &hoffset[0], &hstep[0],
309  &hcount[0], NULL) < 0) {
310 
311  H5Sclose(dspace);
312  H5Dclose(dsetid);
313  HDF5CFUtil::close_fileid(fileid,pass_fileid);
314  ostringstream eherr;
315  eherr << "The selection of hyperslab of the HDF5 dataset " << varname
316  << " fails. "<<endl;
317  throw InternalErr (__FILE__, __LINE__, eherr.str ());
318  }
319 
320  mspace = H5Screate_simple(rank, &hcount[0],NULL);
321  if (mspace < 0) {
322  H5Sclose(dspace);
323  H5Dclose(dsetid);
324  HDF5CFUtil::close_fileid(fileid,pass_fileid);
325  ostringstream eherr;
326  eherr << "The creation of the memory space of the HDF5 dataset " << varname
327  << " fails. "<<endl;
328  throw InternalErr (__FILE__, __LINE__, eherr.str ());
329  }
330 
331 
332  if ((dtypeid = H5Dget_type(dsetid)) < 0) {
333 
334  H5Sclose(mspace);
335  H5Sclose(dspace);
336  H5Dclose(dsetid);
337  HDF5CFUtil::close_fileid(fileid,pass_fileid);
338  ostringstream eherr;
339  eherr << "Obtaining the datatype of the HDF5 dataset " << varname
340  << " fails. "<<endl;
341  throw InternalErr (__FILE__, __LINE__, eherr.str ());
342 
343  }
344 
345  if ((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
346 
347  H5Sclose(mspace);
348  H5Tclose(dtypeid);
349  H5Sclose(dspace);
350  H5Dclose(dsetid);
351  HDF5CFUtil::close_fileid(fileid,pass_fileid);
352  ostringstream eherr;
353  eherr << "Obtaining the memory type of the HDF5 dataset " << varname
354  << " fails. "<<endl;
355  throw InternalErr (__FILE__, __LINE__, eherr.str ());
356 
357  }
358 
359  hid_t read_ret = -1;
360 
361  // Before reading the data, we will check if the memory cache is turned on,
362  // The add_mem_cache is only true when the data memory cache keys are on and used.
363  if(true == add_mem_cache) {
364  if(buf== NULL) {
365  H5Sclose(mspace);
366  H5Tclose(dtypeid);
367  H5Sclose(dspace);
368  H5Dclose(dsetid);
369  HDF5CFUtil::close_fileid(fileid,pass_fileid);
370  throw InternalErr(__FILE__,__LINE__,"The memory data cache buffer needs to be set");
371  }
372  read_ret= H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf);
373  if(read_ret <0){
374  H5Sclose(mspace);
375  H5Tclose(dtypeid);
376  H5Sclose(dspace);
377  H5Dclose(dsetid);
378  HDF5CFUtil::close_fileid(fileid,pass_fileid);
379  throw InternalErr(__FILE__,__LINE__,"Cannot read the data to the buffer.");
380  }
381  }
382 
383 
384  // Now reading the data, note dtype is not dtypeid.
385  // dtype is an enum defined by the handler.
386 
387  switch (dtype) {
388 
389  case H5CHAR:
390  {
391 
392  vector<char> val;
393  val.resize(nelms);
394 
395  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
396  if (read_ret < 0) {
397 
398  H5Sclose(mspace);
399  H5Tclose(memtype);
400  H5Tclose(dtypeid);
401  H5Sclose(dspace);
402  H5Dclose(dsetid);
403  HDF5CFUtil::close_fileid(fileid,pass_fileid);
404  ostringstream eherr;
405  eherr << "Cannot read the HDF5 dataset " << varname
406  << " with the type of H5T_NATIVE_CHAR "<<endl;
407  throw InternalErr (__FILE__, __LINE__, eherr.str ());
408 
409  }
410 
411  if(is_dap4 == true)
412  set_value((dods_int8 *)&val[0],nelms);
413  else {
414 
415  vector<short>newval;
416  newval.resize(nelms);
417 
418  for (int counter = 0; counter < nelms; counter++)
419  newval[counter] = (short) (val[counter]);
420 
421  set_value ((dods_int16 *) &newval[0], nelms);
422  }
423 
424  if(true == data_to_disk_cache) {
425  try {
426  BESDEBUG("h5","writing data to disk cache "<<endl);
427  write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,2,val,nelms);
428  }
429  catch(...) {
430  H5Sclose(mspace);
431  H5Tclose(memtype);
432  H5Tclose(dtypeid);
433  H5Sclose(dspace);
434  H5Dclose(dsetid);
435  HDF5CFUtil::close_fileid(fileid,pass_fileid);
436  ostringstream eherr;
437  eherr << "write data to cache failed.";
438  throw InternalErr (__FILE__, __LINE__, eherr.str ());
439 
440  }
441  }
442 
443  } // case H5CHAR
444  break;
445 
446  // Note: for DAP2, H5INT64,H5UINT64 will be ignored.
447  case H5UCHAR:
448  case H5UINT16:
449  case H5INT16:
450  case H5INT32:
451  case H5UINT32:
452  case H5INT64:
453  case H5UINT64:
454  case H5FLOAT32:
455  case H5FLOAT64:
456 
457 
458  {
459  size_t dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
460  vector<char> val;
461  val.resize(nelms*dtype_size);
462 
463  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
464  if (read_ret < 0) {
465  H5Sclose(mspace);
466  H5Tclose(memtype);
467  H5Tclose(dtypeid);
468  H5Sclose(dspace);
469  H5Dclose(dsetid);
470  HDF5CFUtil::close_fileid(fileid,pass_fileid);
471  ostringstream eherr;
472  eherr << "Cannot read the HDF5 dataset " << varname
473  << " with the type of H5T_NATIVE_UCHAR "<<endl;
474  throw InternalErr (__FILE__, __LINE__, eherr.str ());
475 
476  }
477  // Not sure if "set_value ((dods_byte *) &val[0], nelms);" works.
478  val2buf(&val[0]);
479  set_read_p(true);
480 
481  if(true == data_to_disk_cache) {
482  BESDEBUG("h5","writing data to disk cache "<<endl);
483  try {
484  write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,dtype_size,val,nelms);
485  }
486  catch(...) {
487  H5Sclose(mspace);
488  H5Tclose(memtype);
489  H5Tclose(dtypeid);
490  H5Sclose(dspace);
491  H5Dclose(dsetid);
492  HDF5CFUtil::close_fileid(fileid,pass_fileid);
493  ostringstream eherr;
494  eherr << "write data to cache failed.";
495  throw InternalErr (__FILE__, __LINE__, eherr.str ());
496 
497  }
498 
499  }
500  } // case H5UCHAR...
501  break;
502 
503 
504 
505 #if 0
506  case H5INT16:
507  {
508  vector<short>val;
509  val.resize(nelms);
510 
511  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
512  if (read_ret < 0) {
513 
514  H5Sclose(mspace);
515  H5Tclose(memtype);
516  H5Tclose(dtypeid);
517  H5Sclose(dspace);
518  H5Dclose(dsetid);
519  HDF5CFUtil::close_fileid(fileid,pass_fileid);
520  //H5Fclose(fileid);
521  ostringstream eherr;
522  eherr << "Cannot read the HDF5 dataset " << varname
523  << " with the type of H5T_NATIVE_SHORT "<<endl;
524  throw InternalErr (__FILE__, __LINE__, eherr.str ());
525 
526  }
527  set_value ((dods_int16 *) &val[0], nelms);
528  }// H5INT16
529  break;
530 
531 
532  case H5UINT16:
533  {
534  vector<unsigned short> val;
535  val.resize(nelms);
536  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
537  if (read_ret < 0) {
538 
539  H5Sclose(mspace);
540  H5Tclose(memtype);
541  H5Tclose(dtypeid);
542  H5Sclose(dspace);
543  H5Dclose(dsetid);
544  HDF5CFUtil::close_fileid(fileid,pass_fileid);
545  ostringstream eherr;
546  eherr << "Cannot read the HDF5 dataset " << varname
547  << " with the type of H5T_NATIVE_USHORT "<<endl;
548  throw InternalErr (__FILE__, __LINE__, eherr.str ());
549 
550  }
551  set_value ((dods_uint16 *) &val[0], nelms);
552  } // H5UINT16
553  break;
554 
555 
556  case H5INT32:
557  {
558  vector<int>val;
559  val.resize(nelms);
560  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
561  if (read_ret < 0) {
562  H5Sclose(mspace);
563  H5Tclose(memtype);
564  H5Tclose(dtypeid);
565  H5Sclose(dspace);
566  H5Dclose(dsetid);
567  HDF5CFUtil::close_fileid(fileid,pass_fileid);
568  ostringstream eherr;
569  eherr << "Cannot read the HDF5 dataset " << varname
570  << " with the type of H5T_NATIVE_INT "<<endl;
571  throw InternalErr (__FILE__, __LINE__, eherr.str ());
572 
573  }
574  set_value ((dods_int32 *) &val[0], nelms);
575  } // case H5INT32
576  break;
577 
578  case H5UINT32:
579  {
580  vector<unsigned int>val;
581  val.resize(nelms);
582  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
583  if (read_ret < 0) {
584  H5Sclose(mspace);
585  H5Tclose(memtype);
586  H5Tclose(dtypeid);
587  H5Sclose(dspace);
588  H5Dclose(dsetid);
589  HDF5CFUtil::close_fileid(fileid,pass_fileid);
590  ostringstream eherr;
591  eherr << "Cannot read the HDF5 dataset " << varname
592  << " with the type of H5T_NATIVE_UINT "<<endl;
593  throw InternalErr (__FILE__, __LINE__, eherr.str ());
594 
595  }
596  set_value ((dods_uint32 *) &val[0], nelms);
597  }
598  break;
599 
600  case H5FLOAT32:
601  {
602 
603  vector<float>val;
604  val.resize(nelms);
605 
606  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
607  if (read_ret < 0) {
608  H5Sclose(mspace);
609  H5Tclose(memtype);
610  H5Tclose(dtypeid);
611  H5Sclose(dspace);
612  H5Dclose(dsetid);
613  HDF5CFUtil::close_fileid(fileid,pass_fileid);
614  ostringstream eherr;
615  eherr << "Cannot read the HDF5 dataset " << varname
616  << " with the type of H5T_NATIVE_FLOAT "<<endl;
617  throw InternalErr (__FILE__, __LINE__, eherr.str ());
618 
619  }
620  set_value ((dods_float32 *) &val[0], nelms);
621  }
622  break;
623 
624 
625  case H5FLOAT64:
626  {
627 
628  vector<double>val;
629  val.resize(nelms);
630  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
631 
632  if (read_ret < 0) {
633  H5Sclose(mspace);
634  H5Tclose(memtype);
635  H5Tclose(dtypeid);
636  H5Sclose(dspace);
637  H5Dclose(dsetid);
638  HDF5CFUtil::close_fileid(fileid,pass_fileid);
639  ostringstream eherr;
640  eherr << "Cannot read the HDF5 dataset " << varname
641  << " with the type of H5T_NATIVE_DOUBLE "<<endl;
642  throw InternalErr (__FILE__, __LINE__, eherr.str ());
643 
644  }
645  set_value ((dods_float64 *) &val[0], nelms);
646  } // case H5FLOAT64
647  break;
648 
649 #endif
650 
651  case H5FSTRING:
652  {
653  size_t ty_size = H5Tget_size(dtypeid);
654  if (ty_size == 0) {
655  H5Sclose(mspace);
656  H5Tclose(memtype);
657  H5Tclose(dtypeid);
658  H5Sclose(dspace);
659  H5Dclose(dsetid);
660  HDF5CFUtil::close_fileid(fileid,pass_fileid);
661  ostringstream eherr;
662  eherr << "Cannot obtain the size of the fixed size HDF5 string of the dataset "
663  << varname <<endl;
664  throw InternalErr (__FILE__, __LINE__, eherr.str ());
665  }
666 
667  vector <char> strval;
668  strval.resize(nelms*ty_size);
669  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(void*)&strval[0]);
670 
671  if (read_ret < 0) {
672  H5Sclose(mspace);
673  H5Tclose(memtype);
674  H5Tclose(dtypeid);
675  H5Sclose(dspace);
676  H5Dclose(dsetid);
677  HDF5CFUtil::close_fileid(fileid,pass_fileid);
678  ostringstream eherr;
679  eherr << "Cannot read the HDF5 dataset " << varname
680  << " with the type of the fixed size HDF5 string "<<endl;
681  throw InternalErr (__FILE__, __LINE__, eherr.str ());
682  }
683 
684  string total_string(strval.begin(),strval.end());
685  strval.clear();//save some memory;
686  vector <string> finstrval;
687  finstrval.resize(nelms);
688  for (int i = 0; i<nelms; i++)
689  finstrval[i] = total_string.substr(i*ty_size,ty_size);
690 
691  // Check if we should drop the long string
692 
693  // If the size of an individual element is longer than the current netCDF JAVA
694  // string and the "EnableDropLongString" key is turned on,
695  // No string is generated.
696  if ((true == HDF5RequestHandler::get_drop_long_string()) &&
697  ty_size > NC_JAVA_STR_SIZE_LIMIT) {
698  for (int i = 0; i<nelms; i++)
699  finstrval[i] = "";
700  }
701  set_value(finstrval,nelms);
702  total_string.clear();
703  }
704  break;
705 
706 
707  case H5VSTRING:
708  {
709  size_t ty_size = H5Tget_size(memtype);
710  if (ty_size == 0) {
711  H5Sclose(mspace);
712  H5Tclose(memtype);
713  H5Tclose(dtypeid);
714  H5Sclose(dspace);
715  H5Dclose(dsetid);
716  HDF5CFUtil::close_fileid(fileid,pass_fileid);
717  ostringstream eherr;
718  eherr << "Cannot obtain the size of the fixed size HDF5 string of the dataset "
719  << varname <<endl;
720  throw InternalErr (__FILE__, __LINE__, eherr.str ());
721  }
722  vector <char> strval;
723  strval.resize(nelms*ty_size);
724  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(void*)&strval[0]);
725 
726  if (read_ret < 0) {
727  H5Sclose(mspace);
728  H5Tclose(memtype);
729  H5Tclose(dtypeid);
730  H5Sclose(dspace);
731  H5Dclose(dsetid);
732  HDF5CFUtil::close_fileid(fileid,pass_fileid);
733  ostringstream eherr;
734  eherr << "Cannot read the HDF5 dataset " << varname
735  << " with the type of the HDF5 variable length string "<<endl;
736  throw InternalErr (__FILE__, __LINE__, eherr.str ());
737  }
738 
739  vector<string>finstrval;
740  finstrval.resize(nelms);
741  char*temp_bp = &strval[0];
742  char*onestring = NULL;
743  for (int i =0;i<nelms;i++) {
744  onestring = *(char**)temp_bp;
745  if(onestring!=NULL )
746  finstrval[i] =string(onestring);
747 
748  else // We will add a NULL if onestring is NULL.
749  finstrval[i]="";
750  temp_bp +=ty_size;
751  }
752 
753  if (false == strval.empty()) {
754  herr_t ret_vlen_claim;
755  ret_vlen_claim = H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(void*)&strval[0]);
756  if (ret_vlen_claim < 0){
757  H5Sclose(mspace);
758  H5Tclose(memtype);
759  H5Tclose(dtypeid);
760  H5Sclose(dspace);
761  H5Dclose(dsetid);
762  HDF5CFUtil::close_fileid(fileid,pass_fileid);
763  ostringstream eherr;
764  eherr << "Cannot reclaim the memory buffer of the HDF5 variable length string of the dataset "
765  << varname <<endl;
766  throw InternalErr (__FILE__, __LINE__, eherr.str ());
767 
768  }
769  }
770 
771  // If the size of one string element is longer than the current netCDF JAVA
772  // string and the "EnableDropLongString" key is turned on,
773  // No string is generated.
774  if (true == HDF5RequestHandler::get_drop_long_string()) {
775  bool drop_long_str = false;
776  for (int i =0;i<nelms;i++) {
777  if(finstrval[i].size() >NC_JAVA_STR_SIZE_LIMIT){
778  drop_long_str = true;
779  break;
780  }
781  }
782  if (drop_long_str == true) {
783  for (int i =0;i<nelms;i++)
784  finstrval[i] = "";
785  }
786  }
787  set_value(finstrval,nelms);
788 
789  }
790  break;
791 
792  default:
793  {
794  H5Tclose(memtype);
795  H5Tclose(dtypeid);
796  H5Sclose(mspace);
797  H5Sclose(dspace);
798  H5Dclose(dsetid);
799  HDF5CFUtil::close_fileid(fileid,pass_fileid);
800  ostringstream eherr;
801  eherr << "Cannot read the HDF5 dataset " << varname
802  << " with the unsupported HDF5 datatype"<<endl;
803  throw InternalErr (__FILE__, __LINE__, eherr.str ());
804  }
805  }
806 
807  H5Tclose(memtype);
808  H5Tclose(dtypeid);
809  H5Sclose(mspace);
810  H5Sclose(dspace);
811  H5Dclose(dsetid);
812  HDF5CFUtil::close_fileid(fileid,pass_fileid);
813 
814  return;
815 }
816 
817 bool HDF5CFArray::valid_disk_cache() {
818 
819  bool ret_value = false;
820  if(true == HDF5RequestHandler::get_use_disk_cache()) {
821 
822  BESDEBUG("h5","Coming to disk cache "<<endl);
823  // Check if this is a valid numeric datatype we want to support
824  if(dtype == H5CHAR || dtype ==H5UCHAR || dtype==H5INT16 || dtype ==H5UINT16 ||
825  dtype == H5INT32 || dtype ==H5UINT32 || dtype ==H5FLOAT32 || dtype==H5FLOAT64 ||
826  dtype == H5INT64 || dtype ==H5UINT64){
827 
828  BESDEBUG("h5","Coming to disk cache datatype block"<<endl);
829 
830  string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
831  string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
832  long diskcache_size = HDF5RequestHandler::get_disk_cache_size();
833 
834  if(("" == diskcache_dir)||(""==diskcache_prefix)||(diskcache_size <=0))
835  throw InternalErr (__FILE__, __LINE__, "Either the cached dir is empty or the prefix is NULL or the cache size is not set.");
836  else {
837  struct stat sb;
838  if(stat(diskcache_dir.c_str(),&sb) !=0) {
839  string err_mesg="The cached directory " + diskcache_dir;
840  err_mesg = err_mesg + " doesn't exist. ";
841  throw InternalErr(__FILE__,__LINE__,err_mesg);
842  }
843  else {
844  if(true == S_ISDIR(sb.st_mode)) {
845  if(access(diskcache_dir.c_str(),R_OK|W_OK|X_OK) == -1) {
846  string err_mesg="The cached directory " + diskcache_dir;
847  err_mesg = err_mesg + " can NOT be read,written or executable.";
848  throw InternalErr(__FILE__,__LINE__,err_mesg);
849  }
850  }
851  else {
852  string err_mesg="The cached directory " + diskcache_dir;
853  err_mesg = err_mesg + " is not a directory.";
854  throw InternalErr(__FILE__,__LINE__,err_mesg);
855  }
856  }
857  }
858 
859  short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
860  // Check if we only need to cache the specific compressed dat
861  if(true == HDF5RequestHandler::get_disk_cache_comp_data()){
862  BESDEBUG("h5","Compression disk cache key is true"<<endl);
863  ret_value = valid_disk_cache_for_compressed_data(dtype_size);
864  BESDEBUG("h5","variable disk cache passes the compression parameter check"<<endl);
865  }
866  else {
867  BESDEBUG("h5","Compression disk cache key is NOT set, disk cache key is true."<<endl);
868  ret_value = true;
869  }
870 
871  }
872 
873  }
874  return ret_value;
875 }
876 
877 bool HDF5CFArray:: valid_disk_cache_for_compressed_data(short dtype_size) const {
878 
879  bool ret_value = false;
880  // The compression ratio should be smaller then the threshold(hard to compress)
881  // and the total var size should be bigger than the defined size(bigger)
882 #if 0
883  size_t total_byte = total_elems*dtype_size;
884 #endif
885  if((comp_ratio < HDF5RequestHandler::get_disk_comp_threshold())
886  && (total_elems*dtype_size >= HDF5RequestHandler::get_disk_var_size())) {
887  if( true == HDF5RequestHandler::get_disk_cache_float_only_comp()) {
888  if(dtype==H5FLOAT32 || dtype == H5FLOAT64)
889  ret_value = true;
890  }
891  else
892  ret_value = true;
893  }
894  return ret_value;
895 
896 }
897 
898 bool HDF5CFArray::obtain_cached_data(HDF5DiskCache *disk_cache,const string & cache_fpath, int fd,vector<int> &cd_step, vector<int>&cd_count,size_t total_read,short dtype_size) {
899 
900  ssize_t ret_read_val = -1;
901  vector<char>buf;
902 
903  buf.resize(total_read);
904  ret_read_val = HDF5CFUtil::read_buffer_from_file(fd,(void*)&buf[0],total_read);
905  disk_cache->unlock_and_close(cache_fpath);
906  if((-1 == ret_read_val) || (ret_read_val != (ssize_t)total_read)) {
907  disk_cache->purge_file(cache_fpath);
908  return false;
909  }
910  else {
911  unsigned int nele_to_read = 1;
912  for(int i = 0; i<rank;i++)
913  nele_to_read *=cd_count[i];
914 
915  if(nele_to_read == (total_read/dtype_size)) {
916  val2buf(&buf[0]);
917  set_read_p(true);
918  }
919  else { // Need to re-assemble the buffer according to different datatype
920 
921  vector<int>cd_start(rank,0);
922  vector<size_t>cd_pos(rank,0);
923  int nelms_to_send = 1;
924  for(int i = 0; i <rank; i++)
925  nelms_to_send = nelms_to_send*cd_count[i];
926 
927  switch (dtype) {
928 
929  case H5CHAR:
930  {
931 #if 0
932  vector<int>total_val;
933  total_val.resize(total_read/dtype_size);
934  memcpy(&total_val[0],(void*)&buf[0],total_read);
935 
936  vector<int>final_val;
937  subset<int>(
938  &total_val[0],
939  rank,
940  dimsizes,
941  &cd_start[0],
942  &cd_step[0],
943  &cd_count[0],
944  &final_val,
945  cd_pos,
946  0
947  );
948 
949 #endif
950 
951  if(is_dap4 == false) {
952  vector<short>final_val;
953  subset<short>(
954  &buf[0],
955  rank,
956  dimsizes,
957  &cd_start[0],
958  &cd_step[0],
959  &cd_count[0],
960  &final_val,
961  cd_pos,
962  0
963  );
964  set_value((dods_int16*)&final_val[0],nelms_to_send);
965  }
966  else {
967  vector<char>final_val;
968  subset<char>(
969  &buf[0],
970  rank,
971  dimsizes,
972  &cd_start[0],
973  &cd_step[0],
974  &cd_count[0],
975  &final_val,
976  cd_pos,
977  0
978  );
979  set_value((dods_int8*)&final_val[0],nelms_to_send);
980  }
981 
982  }
983 
984  break;
985  case H5UCHAR:
986  {
987 #if 0
988  vector<unsigned char>total_val;
989  total_val.resize(total_read/dtype_size);
990  memcpy(&total_val[0],(void*)&buf[0],total_read);
991 
992  vector<unsigned char>final_val;
993  subset<unsigned char>(
994  &total_val[0],
995  rank,
996  dimsizes,
997  &cd_start[0],
998  &cd_step[0],
999  &cd_count[0],
1000  &final_val,
1001  cd_pos,
1002  0
1003  );
1004 
1005 #endif
1006  vector<unsigned char>final_val;
1007  subset<unsigned char>(
1008  &buf[0],
1009  rank,
1010  dimsizes,
1011  &cd_start[0],
1012  &cd_step[0],
1013  &cd_count[0],
1014  &final_val,
1015  cd_pos,
1016  0
1017  );
1018 
1019  set_value ((dods_byte *) &final_val[0], nelms_to_send);
1020  }
1021  break;
1022 
1023  case H5INT16:
1024  {
1025 #if 0
1026  vector<short>total_val;
1027  total_val.resize(total_read/dtype_size);
1028  memcpy(&total_val[0],(void*)&buf[0],total_read);
1029 
1030  vector<short>final_val;
1031  subset<short>(
1032  &total_val[0],
1033  rank,
1034  dimsizes,
1035  &cd_start[0],
1036  &cd_step[0],
1037  &cd_count[0],
1038  &final_val,
1039  cd_pos,
1040  0
1041  );
1042 #endif
1043 
1044  vector<short>final_val;
1045  subset<short>(
1046  &buf[0],
1047  rank,
1048  dimsizes,
1049  &cd_start[0],
1050  &cd_step[0],
1051  &cd_count[0],
1052  &final_val,
1053  cd_pos,
1054  0
1055  );
1056 
1057  set_value ((dods_int16 *) &final_val[0], nelms_to_send);
1058  }
1059  break;
1060 
1061  case H5UINT16:
1062  {
1063 #if 0
1064  vector<unsigned short>total_val;
1065  total_val.resize(total_read/dtype_size);
1066  memcpy(&total_val[0],(void*)&buf[0],total_read);
1067 
1068  vector<unsigned short>final_val;
1069  subset<unsigned short>(
1070  &total_val[0],
1071  rank,
1072  dimsizes,
1073  &cd_start[0],
1074  &cd_step[0],
1075  &cd_count[0],
1076  &final_val,
1077  cd_pos,
1078  0
1079  );
1080 #endif
1081 
1082  vector<unsigned short>final_val;
1083  subset<unsigned short>(
1084  &buf[0],
1085  rank,
1086  dimsizes,
1087  &cd_start[0],
1088  &cd_step[0],
1089  &cd_count[0],
1090  &final_val,
1091  cd_pos,
1092  0
1093  );
1094 
1095  set_value ((dods_uint16 *) &final_val[0], nelms_to_send);
1096  }
1097  break;
1098 
1099  case H5INT32:
1100  {
1101 #if 0
1102  vector<int>total_val;
1103  total_val.resize(total_read/dtype_size);
1104  memcpy(&total_val[0],(void*)&buf[0],total_read);
1105 
1106  vector<int>final_val;
1107  subset<int>(
1108  &total_val[0],
1109  rank,
1110  dimsizes,
1111  &cd_start[0],
1112  &cd_step[0],
1113  &cd_count[0],
1114  &final_val,
1115  cd_pos,
1116  0
1117  );
1118 
1119 #endif
1120 
1121  vector<int>final_val;
1122  subset<int>(
1123  &buf[0],
1124  rank,
1125  dimsizes,
1126  &cd_start[0],
1127  &cd_step[0],
1128  &cd_count[0],
1129  &final_val,
1130  cd_pos,
1131  0
1132  );
1133 
1134 
1135  set_value ((dods_int32 *) &final_val[0], nelms_to_send);
1136  }
1137  break;
1138 
1139  case H5UINT32:
1140  {
1141 #if 0
1142  vector<unsigned int>total_val;
1143  total_val.resize(total_read/dtype_size);
1144  memcpy(&total_val[0],(void*)&buf[0],total_read);
1145 
1146  vector<unsigned int>final_val;
1147  subset<unsigned int>(
1148  &total_val[0],
1149  rank,
1150  dimsizes,
1151  &cd_start[0],
1152  &cd_step[0],
1153  &cd_count[0],
1154  &final_val,
1155  cd_pos,
1156  0
1157  );
1158 #endif
1159 
1160  vector<unsigned int>final_val;
1161  subset<unsigned int>(
1162  &buf[0],
1163  rank,
1164  dimsizes,
1165  &cd_start[0],
1166  &cd_step[0],
1167  &cd_count[0],
1168  &final_val,
1169  cd_pos,
1170  0
1171  );
1172 
1173  set_value ((dods_uint32 *) &final_val[0], nelms_to_send);
1174  }
1175  break;
1176 
1177  case H5INT64: // Only for DAP4 CF
1178  {
1179 #if 0
1180  vector<unsigned int>total_val;
1181  total_val.resize(total_read/dtype_size);
1182  memcpy(&total_val[0],(void*)&buf[0],total_read);
1183 
1184  vector<unsigned int>final_val;
1185  subset<unsigned int>(
1186  &total_val[0],
1187  rank,
1188  dimsizes,
1189  &cd_start[0],
1190  &cd_step[0],
1191  &cd_count[0],
1192  &final_val,
1193  cd_pos,
1194  0
1195  );
1196 #endif
1197 
1198  vector<long long >final_val;
1199  subset<long long >(
1200  &buf[0],
1201  rank,
1202  dimsizes,
1203  &cd_start[0],
1204  &cd_step[0],
1205  &cd_count[0],
1206  &final_val,
1207  cd_pos,
1208  0
1209  );
1210 
1211  set_value ((dods_int64 *) &final_val[0], nelms_to_send);
1212  }
1213  break;
1214 
1215 
1216 
1217  case H5UINT64: // Only for DAP4 CF
1218  {
1219 #if 0
1220  vector<unsigned int>total_val;
1221  total_val.resize(total_read/dtype_size);
1222  memcpy(&total_val[0],(void*)&buf[0],total_read);
1223 
1224  vector<unsigned int>final_val;
1225  subset<unsigned int>(
1226  &total_val[0],
1227  rank,
1228  dimsizes,
1229  &cd_start[0],
1230  &cd_step[0],
1231  &cd_count[0],
1232  &final_val,
1233  cd_pos,
1234  0
1235  );
1236 #endif
1237 
1238  vector<unsigned long long >final_val;
1239  subset<unsigned long long >(
1240  &buf[0],
1241  rank,
1242  dimsizes,
1243  &cd_start[0],
1244  &cd_step[0],
1245  &cd_count[0],
1246  &final_val,
1247  cd_pos,
1248  0
1249  );
1250 
1251  set_value ((dods_uint64 *) &final_val[0], nelms_to_send);
1252  }
1253  break;
1254 
1255 
1256  case H5FLOAT32:
1257  {
1258 #if 0
1259  vector<float>total_val;
1260  total_val.resize(total_read/dtype_size);
1261  memcpy(&total_val[0],(void*)&buf[0],total_read);
1262 
1263  vector<float>final_val;
1264  subset<float>(
1265  &total_val[0],
1266  rank,
1267  dimsizes,
1268  &cd_start[0],
1269  &cd_step[0],
1270  &cd_count[0],
1271  &final_val,
1272  cd_pos,
1273  0
1274  );
1275 #endif
1276 
1277  vector<float>final_val;
1278  subset<float>(
1279  &buf[0],
1280  rank,
1281  dimsizes,
1282  &cd_start[0],
1283  &cd_step[0],
1284  &cd_count[0],
1285  &final_val,
1286  cd_pos,
1287  0
1288  );
1289 
1290 
1291  set_value ((dods_float32 *) &final_val[0], nelms_to_send);
1292  }
1293  break;
1294  case H5FLOAT64:
1295  {
1296 #if 0
1297  vector<double>total_val;
1298  total_val.resize(total_read/dtype_size);
1299  memcpy(&total_val[0],(void*)&buf[0],total_read);
1300 
1301  vector<double>final_val;
1302  subset<double>(
1303  &total_val[0],
1304  rank,
1305  dimsizes,
1306  &cd_start[0],
1307  &cd_step[0],
1308  &cd_count[0],
1309  &final_val,
1310  cd_pos,
1311  0
1312  );
1313 #endif
1314  vector<double>final_val;
1315  subset<double>(
1316  &buf[0],
1317  rank,
1318  dimsizes,
1319  &cd_start[0],
1320  &cd_step[0],
1321  &cd_count[0],
1322  &final_val,
1323  cd_pos,
1324  0
1325  );
1326 
1327  set_value ((dods_float64 *) &final_val[0], nelms_to_send);
1328  }
1329  break;
1330  default:
1331  throw InternalErr (__FILE__, __LINE__, "unsupported data type.");
1332 
1333  }// "end switch(dtype)"
1334  }// "end else (stride is not 1)"
1335  return true;
1336  }// "end else(full_read = true)"
1337 }
1338 
1339 
1340 void
1341 HDF5CFArray::write_data_to_cache(hid_t dset_id, hid_t /*dspace_id*/, hid_t /*mspace_id*/, hid_t memtype,
1342  const string& cache_fpath, short dtype_size, const vector<char> &buf, int nelms) {
1343 
1344  unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
1345  string disk_cache_dir = HDF5RequestHandler::get_disk_cache_dir();
1346  string disk_cache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
1347  HDF5DiskCache *disk_cache = HDF5DiskCache::get_instance(disk_cache_size,disk_cache_dir,disk_cache_prefix);
1348  int total_nelem = 1;
1349  for(int i = 0; i <rank; i++)
1350  total_nelem = total_nelem*dimsizes[i];
1351 
1352  vector<char>val;
1353 
1354  if(H5CHAR == dtype && is_dap4 == false) {
1355 
1356  vector<short>newval;
1357  newval.resize(total_nelem);
1358  if(total_nelem == nelms) {
1359  for (int i = 0; i < total_nelem;i++)
1360  newval[i] = (short)buf[i];
1361  disk_cache->write_cached_data2(cache_fpath,sizeof(short)*total_nelem,(const void*)&newval[0]);
1362  }
1363  else {
1364  vector<char>val2;
1365  val2.resize(total_nelem);
1366  if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val2[0])<0)
1367  throw InternalErr (__FILE__, __LINE__, "Cannot read the whole HDF5 dataset for the disk cache.");
1368  for (int i = 0; i < total_nelem;i++)
1369  newval[i] = (short)val2[i];
1370  disk_cache->write_cached_data2(cache_fpath,sizeof(short)*total_nelem,(const void*)&newval[0]);
1371  }
1372  }
1373  else {
1374  if(total_nelem == nelms) {
1375  disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(const void*)&buf[0]);
1376  }
1377  else {
1378  val.resize(dtype_size*total_nelem);
1379  if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val[0])<0)
1380  throw InternalErr (__FILE__, __LINE__, "Cannot read the whole SDS for cache.");
1381 
1382  disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(const void*)&val[0]);
1383  }
1384  }
1385 }
1386 
1387 #if 0
1388 void HDF5CFArray::read_data_from_mem_cache(void*buf) {
1389 
1390  vector<int>offset;
1391  vector<int>count;
1392  vector<int>step;
1393  int nelms = format_constraint (&offset[0], &step[0], &count[0]);
1394  // set the original position to the starting point
1395  vector<int>at_pos(at_ndims,0);
1396  for (int i = 0; i< rank; i++)
1397  at_pos[i] = at_offset[i];
1398 
1399 
1400  switch (dtype) {
1401 
1402  case H5UCHAR:
1403 
1404  {
1405  vector<unsigned char> val;
1406  val.resize(nelms);
1407  subset<unsigned char>(
1408  &total_val[0],
1409  rank,
1410  dimsizes,
1411  offset,
1412  step,
1413  count,
1414  &final_val,
1415  at_pos,
1416  0
1417  );
1418 
1419 
1420  set_value ((dods_byte *) &val[0], nelms);
1421  } // case H5UCHAR
1422  break;
1423 
1424 
1425  case H5CHAR:
1426  {
1427 
1428  vector<char> val;
1429  val.resize(nelms);
1430 
1431  if (0 == rank)
1432  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1433  else
1434  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1435 
1436  if (read_ret < 0) {
1437 
1438  if (rank > 0)
1439  H5Sclose(mspace);
1440  H5Tclose(memtype);
1441  H5Tclose(dtypeid);
1442  H5Sclose(dspace);
1443  H5Dclose(dsetid);
1444  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1445  ostringstream eherr;
1446  eherr << "Cannot read the HDF5 dataset " << varname
1447  << " with the type of H5T_NATIVE_CHAR "<<endl;
1448  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1449 
1450  }
1451 
1452  vector<short>newval;
1453  newval.resize(nelms);
1454 
1455  for (int counter = 0; counter < nelms; counter++)
1456  newval[counter] = (short) (val[counter]);
1457 
1458  set_value ((dods_int16 *) &newval[0], nelms);
1459  } // case H5CHAR
1460  break;
1461 
1462 
1463  case H5INT16:
1464  {
1465  vector<short>val;
1466  val.resize(nelms);
1467 
1468  if (0 == rank)
1469  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1470  else
1471  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1472 
1473  if (read_ret < 0) {
1474 
1475  if (rank > 0)
1476  H5Sclose(mspace);
1477  H5Tclose(memtype);
1478  H5Tclose(dtypeid);
1479  H5Sclose(dspace);
1480  H5Dclose(dsetid);
1481  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1482  //H5Fclose(fileid);
1483  ostringstream eherr;
1484  eherr << "Cannot read the HDF5 dataset " << varname
1485  << " with the type of H5T_NATIVE_SHORT "<<endl;
1486  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1487 
1488  }
1489  set_value ((dods_int16 *) &val[0], nelms);
1490  }// H5INT16
1491  break;
1492 
1493 
1494  case H5UINT16:
1495  {
1496  vector<unsigned short> val;
1497  val.resize(nelms);
1498  if (0 == rank)
1499  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1500  else
1501  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1502 
1503  if (read_ret < 0) {
1504 
1505  if (rank > 0) H5Sclose(mspace);
1506  H5Tclose(memtype);
1507  H5Tclose(dtypeid);
1508  H5Sclose(dspace);
1509  H5Dclose(dsetid);
1510  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1511  ostringstream eherr;
1512  eherr << "Cannot read the HDF5 dataset " << varname
1513  << " with the type of H5T_NATIVE_USHORT "<<endl;
1514  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1515 
1516  }
1517  set_value ((dods_uint16 *) &val[0], nelms);
1518  } // H5UINT16
1519  break;
1520 
1521 
1522  case H5INT32:
1523  {
1524  vector<int>val;
1525  val.resize(nelms);
1526  if (0 == rank)
1527  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1528  else
1529  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1530 
1531  if (read_ret < 0) {
1532  if (rank > 0)
1533  H5Sclose(mspace);
1534  H5Tclose(memtype);
1535  H5Tclose(dtypeid);
1536  H5Sclose(dspace);
1537  H5Dclose(dsetid);
1538  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1539  ostringstream eherr;
1540  eherr << "Cannot read the HDF5 dataset " << varname
1541  << " with the type of H5T_NATIVE_INT "<<endl;
1542  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1543 
1544  }
1545  set_value ((dods_int32 *) &val[0], nelms);
1546  } // case H5INT32
1547  break;
1548 
1549  case H5UINT32:
1550  {
1551  vector<unsigned int>val;
1552  val.resize(nelms);
1553  if (0 == rank)
1554  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1555  else
1556  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1557 
1558  if (read_ret < 0) {
1559 
1560  if (rank > 0)
1561  H5Sclose(mspace);
1562  H5Tclose(memtype);
1563  H5Tclose(dtypeid);
1564  H5Sclose(dspace);
1565  H5Dclose(dsetid);
1566  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1567  ostringstream eherr;
1568  eherr << "Cannot read the HDF5 dataset " << varname
1569  << " with the type of H5T_NATIVE_UINT "<<endl;
1570  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1571 
1572  }
1573  set_value ((dods_uint32 *) &val[0], nelms);
1574  }
1575  break;
1576 
1577  case H5FLOAT32:
1578  {
1579 
1580  vector<float>val;
1581  val.resize(nelms);
1582 
1583  if (0 == rank)
1584  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1585  else
1586  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1587 
1588  if (read_ret < 0) {
1589  if (rank > 0)
1590  H5Sclose(mspace);
1591  H5Tclose(memtype);
1592  H5Tclose(dtypeid);
1593  H5Sclose(dspace);
1594  H5Dclose(dsetid);
1595  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1596  ostringstream eherr;
1597  eherr << "Cannot read the HDF5 dataset " << varname
1598  << " with the type of H5T_NATIVE_FLOAT "<<endl;
1599  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1600 
1601  }
1602  set_value ((dods_float32 *) &val[0], nelms);
1603  }
1604  break;
1605 
1606 
1607  case H5FLOAT64:
1608  {
1609 
1610  vector<double>val;
1611  val.resize(nelms);
1612  if (0 == rank)
1613  read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1614  else
1615  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1616 
1617  if (read_ret < 0) {
1618  if (rank > 0)
1619  H5Sclose(mspace);
1620  H5Tclose(memtype);
1621  H5Tclose(dtypeid);
1622  H5Sclose(dspace);
1623  H5Dclose(dsetid);
1624  HDF5CFUtil::close_fileid(fileid,pass_fileid);
1625  ostringstream eherr;
1626  eherr << "Cannot read the HDF5 dataset " << varname
1627  << " with the type of H5T_NATIVE_DOUBLE "<<endl;
1628  throw InternalErr (__FILE__, __LINE__, eherr.str ());
1629 
1630  }
1631  set_value ((dods_float64 *) &val[0], nelms);
1632  } // case H5FLOAT64
1633  break;
1634 
1635 
1636 
1637  // Just see if it works.
1638  val2buf(buf);
1639  set_read_p(true);
1640  return;
1641 }
1642 #endif
1643 
1644 #if 0
1645 // We don't inherit libdap Array Class's transform_to_dap4 method since it also transforms attributes.
1646 BaseType* HDF5CFArray::h5cfdims_transform_to_dap4(D4Group *grp) {
1647 
1648  if(grp == NULL)
1649  return NULL;
1650  Array *dest = static_cast<HDF5CFArray*>(ptr_duplicate());
1651 
1652  // If there is just a size, don't make
1653  // a D4Dimension (In DAP4 you cannot share a dimension unless it has
1654  // a name). jhrg 3/18/14
1655 
1656  D4Dimensions *grp_dims = grp->dims();
1657  for (Array::Dim_iter dap2_dim = dest->dim_begin(), e = dest->dim_end(); dap2_dim != e; ++dap2_dim) {
1658  if (!(*dap2_dim).name.empty()) {
1659 
1660  // If a D4Dimension with the name already exists, use it.
1661  D4Dimension *d4_dim = grp_dims->find_dim((*dap2_dim).name);
1662  if (!d4_dim) {
1663  d4_dim = new D4Dimension((*dap2_dim).name, (*dap2_dim).size);
1664  grp_dims->add_dim_nocopy(d4_dim);
1665  }
1666  // At this point d4_dim's name and size == those of (*d) so just set
1667  // the D4Dimension pointer so it matches the one in the D4Group.
1668  (*dap2_dim).dim = d4_dim;
1669  }
1670  }
1671 
1672  return dest;
1673 
1674 }
1675 #endif
1676 
1677 // We don't inherit libdap Array Class's transform_to_dap4 method since CF option is still using it.
1678 // This function is used for 64-bit integer mapping to DAP4 for the CF option. largely borrowed from
1679 // DAP4 code.
1680 BaseType* HDF5CFArray::h5cfdims_transform_to_dap4_int64(D4Group *grp) {
1681 
1682  if(grp == NULL)
1683  return NULL;
1684  Array *dest = static_cast<HDF5CFArray*>(ptr_duplicate());
1685 
1686  // If there is just a size, don't make
1687  // a D4Dimension (In DAP4 you cannot share a dimension unless it has
1688  // a name). jhrg 3/18/14
1689 
1690  for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1691  if (false == (*d).name.empty()) {
1692 
1693  D4Group *temp_grp = grp;
1694  D4Dimension *d4_dim = NULL;
1695  while(temp_grp) {
1696 
1697  D4Dimensions *temp_dims = temp_grp->dims();
1698 
1699  // Check if the dimension is defined in this group
1700  d4_dim = temp_dims->find_dim((*d).name);
1701  if(d4_dim) {
1702  (*d).dim = d4_dim;
1703  break;
1704  }
1705 
1706  if(temp_grp->get_parent())
1707  temp_grp = static_cast<D4Group*>(temp_grp->get_parent());
1708  else
1709  temp_grp = 0;
1710 
1711  }
1712 
1713  // Not find this dimension in any of the ancestor groups, add it to this group.
1714  // The following block is fine, but to avoid the complaint from sonarcloud.
1715  // Use a bool.
1716  bool d4_dim_null = ((d4_dim==NULL)?true:false);
1717 #if 0
1718  //if(d4_dim == NULL) {
1719 #endif
1720  // Not find this dimension in any of the ancestor groups, add it to this group.
1721  if(d4_dim_null == true) {
1722 
1723  d4_dim = new D4Dimension((*d).name, (*d).size);
1724  D4Dimensions * dims = grp->dims();
1725  dims->add_dim_nocopy(d4_dim);
1726  (*d).dim = d4_dim;
1727  }
1728  }
1729  }
1730 
1731  dest->set_is_dap4(true);
1732 
1733  return dest;
1734 
1735 }
1736 #if 0
1737 // parse constraint expr. and make hdf5 coordinate point location.
1738 // return number of elements to read.
1739 int
1740 HDF5CFArray::format_constraint (int *offset, int *step, int *count)
1741 {
1742 
1743  long nels = 1;
1744  int id = 0;
1745 
1746  Dim_iter p = dim_begin ();
1747 
1748  while (p != dim_end ()) {
1749 
1750  int start = dimension_start (p, true);
1751  int stride = dimension_stride (p, true);
1752  int stop = dimension_stop (p, true);
1753 
1754  // Check for illegal constraint
1755  if (start > stop) {
1756  ostringstream oss;
1757 
1758  oss << "Array/Grid hyperslab start point "<< start <<
1759  " is greater than stop point " << stop <<".";
1760  throw Error(malformed_expr, oss.str());
1761  }
1762 
1763  offset[id] = start;
1764  step[id] = stride;
1765  count[id] = ((stop - start) / stride) + 1; // count of elements
1766  nels *= count[id]; // total number of values for variable
1767 
1768  BESDEBUG ("h5",
1769  "=format_constraint():"
1770  << "id=" << id << " offset=" << offset[id]
1771  << " step=" << step[id]
1772  << " count=" << count[id]
1773  << endl);
1774 
1775  id++;
1776  p++;
1777  }
1778 
1779  return nels;
1780 }
1781 
1782 #endif
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
include the entry functions to execute the handlers
virtual void unlock_and_close(const std::string &target)
virtual void purge_file(const std::string &file)
Purge a single file from the cache.
virtual void read_data_NOT_from_mem_cache(bool add_cache, void *buf)
Definition: HDF5CFArray.cc:175
static HDF5DiskCache * get_instance(const long, const std::string &, const std::string &)
Helper functions for generating DAS attributes and a function to check BES Key.
static ssize_t read_buffer_from_file(int fd, void *buf, size_t)
Getting a subset of a variable.
Definition: HDF5CFUtil.cc:1195