bes  Updated for version 3.20.10
DmrppRequestHandler.cc
1 // DmrppRequestHandler.cc
2 
3 // Copyright (c) 2016 OPeNDAP, Inc. Author: James Gallagher
4 // <jgallagher@opendap.org>, Patrick West <pwest@opendap.org>
5 // Nathan Potter <npotter@opendap.org>
6 //
7 // modify it under the terms of the GNU Lesser General Public License
8 // as published by the Free Software Foundation; either version 2.1 of
9 // the License, or (at your option) any later version.
10 //
11 // This library is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 // Lesser General Public License for more details.
15 //
16 // License along with this library; if not, write to the Free Software
17 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 // 02110-1301 U\ SA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI.
21 // 02874-0112.
22 
23 #include "config.h"
24 
25 #include <string>
26 #include <memory>
27 #include <sstream>
28 
29 #include <curl/curl.h>
30 
31 #include <libdap/Ancillary.h>
32 #include <libdap/DMR.h>
33 #include <libdap/D4Group.h>
34 #include <libdap/DAS.h>
35 
36 #include <libdap/InternalErr.h>
37 #include <libdap/mime_util.h> // for name_path
38 
39 #include <BESResponseHandler.h>
40 #include <BESResponseNames.h>
41 #include <BESDapNames.h>
42 #include <BESDataNames.h>
43 #include <BESDASResponse.h>
44 #include <BESDDSResponse.h>
45 #include <BESDataDDSResponse.h>
46 #include <BESVersionInfo.h>
47 #include <BESContainer.h>
48 #include <ObjMemCache.h>
49 
50 #include <BESDMRResponse.h>
51 
52 #include <BESConstraintFuncs.h>
53 #include <BESServiceRegistry.h>
54 #include <BESUtil.h>
55 #include <BESLog.h>
56 #include <TheBESKeys.h>
57 
58 #include <BESDapError.h>
59 #include <BESInternalFatalError.h>
60 #include <BESDebug.h>
61 #include <BESStopWatch.h>
62 
63 #if 1
64 #define PUGIXML_NO_XPATH
65 #define PUGIXML_HEADER_ONLY
66 #include <pugixml.hpp>
67 #endif
68 
69 #include "DmrppNames.h"
70 //#include "DMRpp.h"
71 #include "DmrppTypeFactory.h"
72 //#include "DmrppParserSax2.h"
73 #include "DmrppRequestHandler.h"
74 #include "CurlHandlePool.h"
75 //#include "DmrppMetadataStore.h"
76 #include "CredentialsManager.h"
77 
78 using namespace bes;
79 using namespace libdap;
80 using namespace std;
81 
82 #define MODULE_NAME "dmrpp_module"
83 #ifndef MODULE_VERSION
84 #define MODULE_VERSION "unset" // Set this in the Makefile.am
85 #endif
86 
87 #define prolog std::string("DmrppRequestHandler::").append(__func__).append("() - ")
88 
89 #define USE_DMZ_TO_MANAGE_XML 1
90 
91 namespace dmrpp {
92 
93 ObjMemCache *DmrppRequestHandler::das_cache = 0;
94 ObjMemCache *DmrppRequestHandler::dds_cache = 0;
95 ObjMemCache *DmrppRequestHandler::dmr_cache = 0;
96 
97 shared_ptr<DMZ> DmrppRequestHandler::dmz(nullptr);
98 
99 // This is used to maintain a pool of reusable curl handles that enable connection
100 // reuse. jhrg
101 CurlHandlePool *DmrppRequestHandler::curl_handle_pool = 0;
102 
103 bool DmrppRequestHandler::d_use_transfer_threads = true;
104 unsigned int DmrppRequestHandler::d_max_transfer_threads = 8;
105 
106 bool DmrppRequestHandler::d_use_compute_threads = true;
107 unsigned int DmrppRequestHandler::d_max_compute_threads = 8;
108 
109 // Default minimum value is 2MB: 2 * (1024*1024)
110 unsigned long long DmrppRequestHandler::d_contiguous_concurrent_threshold = DMRPP_DEFAULT_CONTIGUOUS_CONCURRENT_THRESHOLD;
111 
112 // This behavior mirrors the SAX2 parser behavior where the software doesn't require that
113 // a variable actually have chunks (that is, some variable might not have any data).
114 // We could make this a run-time option if needed. jhrg 11/4/21
115 bool DmrppRequestHandler::d_require_chunks = false;
116 
117 // See the comment in the header for more about this kludge. jhrg 11/9/21
118 bool DmrppRequestHandler::d_emulate_original_filter_order_behavior = false;
119 
120 static void read_key_value(const std::string &key_name, bool &key_value)
121 {
122  bool key_found = false;
123  string value;
124  TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
125  if (key_found) {
126  value = BESUtil::lowercase(value);
127  key_value = (value == "true" || value == "yes");
128  }
129 }
130 
131 static void read_key_value(const std::string &key_name, unsigned int &key_value)
132 {
133  bool key_found = false;
134  string value;
135  TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
136  if (key_found) {
137  istringstream iss(value);
138  iss >> key_value;
139  }
140 }
141 static void read_key_value(const std::string &key_name, unsigned long long &key_value)
142 {
143  bool key_found = false;
144  string value;
145  TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
146  if (key_found) {
147  istringstream iss(value);
148  iss >> key_value;
149  }
150 }
151 
156 DmrppRequestHandler::DmrppRequestHandler(const string &name) :
157  BESRequestHandler(name)
158 {
159  add_method(DMR_RESPONSE, dap_build_dmr);
160  add_method(DAP4DATA_RESPONSE, dap_build_dap4data);
161  add_method(DAS_RESPONSE, dap_build_das);
162  add_method(DDS_RESPONSE, dap_build_dds);
163  add_method(DATA_RESPONSE, dap_build_dap2data);
164 
165  add_method(VERS_RESPONSE, dap_build_vers);
166  add_method(HELP_RESPONSE, dap_build_help);
167 
168  stringstream msg;
169  read_key_value(DMRPP_USE_TRANSFER_THREADS_KEY, d_use_transfer_threads);
170  read_key_value(DMRPP_MAX_TRANSFER_THREADS_KEY, d_max_transfer_threads);
171  msg << prolog << "Concurrent Transfer Threads: ";
172  if(DmrppRequestHandler::d_use_transfer_threads){
173  msg << "Enabled. max_transfer_threads: " << DmrppRequestHandler::d_max_transfer_threads << endl;
174  }
175  else{
176  msg << "Disabled." << endl;
177  }
178 
179  INFO_LOG(msg.str() );
180  msg.str(std::string());
181 
182  read_key_value(DMRPP_USE_COMPUTE_THREADS_KEY, d_use_compute_threads);
183  read_key_value(DMRPP_MAX_COMPUTE_THREADS_KEY, d_max_compute_threads);
184  msg << prolog << "Concurrent Compute Threads: ";
185  if(DmrppRequestHandler::d_use_compute_threads){
186  msg << "Enabled. max_compute_threads: " << DmrppRequestHandler::d_max_compute_threads << endl;
187  }
188  else{
189  msg << "Disabled." << endl;
190  }
191 
192  INFO_LOG(msg.str() );
193  msg.str(std::string());
194 
195  // DMRPP_CONTIGUOUS_CONCURRENT_THRESHOLD_KEY
196  read_key_value(DMRPP_CONTIGUOUS_CONCURRENT_THRESHOLD_KEY, d_contiguous_concurrent_threshold);
197  msg << prolog << "Contiguous Concurrency Threshold: " << d_contiguous_concurrent_threshold << " bytes." << endl;
198  INFO_LOG(msg.str() );
199 
200 #if !HAVE_CURL_MULTI_API
201  if (DmrppRequestHandler::d_use_transfer_threads)
202  ERROR_LOG("The DMR++ handler is configured to use parallel transfers, but the libcurl Multi API is not present, defaulting to serial transfers");
203 #endif
204 
206 
207  if (!curl_handle_pool)
208  curl_handle_pool = new CurlHandlePool(d_max_transfer_threads);
209 
210  // This and the matching cleanup function can be called many times as long as
211  // they are called in balanced pairs. jhrg 9/3/20
212  // TODO 10/8/21 move this into the http at the top level of the BES. That is, all
213  // calls to this should be moved out of handlers and handlers can/should
214  // assume that curl is present and init'd. jhrg
215  curl_global_init(CURL_GLOBAL_DEFAULT);
216 }
217 
218 DmrppRequestHandler::~DmrppRequestHandler()
219 {
220  delete curl_handle_pool;
221  curl_global_cleanup();
222 }
223 
229 void
230 handle_exception(const string &file, int line)
231  try {
232  throw;
233  }
234  catch (BESError &e) {
235  throw e;
236  }
237  catch (InternalErr &e) {
238  throw BESDapError(e.get_error_message(), true, e.get_error_code(), file, line);
239  }
240  catch (Error &e) {
241  throw BESDapError(e.get_error_message(), false, e.get_error_code(), file, line);
242  }
243  catch (std::exception &e) {
244  BESInternalFatalError(string("C++ exception: ").append(e.what()), file, line);
245  }
246  catch (...) {
247  throw BESInternalFatalError("Unknown exception caught building DAP4 Data response", file, line);
248  }
249 
250 
251 void DmrppRequestHandler::build_dmr_from_file(BESContainer *container, DMR* dmr)
252 {
253  string data_pathname = container->access();
254 
255  dmr->set_filename(data_pathname);
256  dmr->set_name(name_path(data_pathname));
257 
258 #if USE_DMZ_TO_MANAGE_XML
259  dmz = shared_ptr<DMZ>(new DMZ);
260 
261  // Enable adding the DMZ to the BaseTypes built by the factory
262  DmrppTypeFactory BaseFactory(dmz);
263  dmr->set_factory(&BaseFactory);
264 
265  dmz->parse_xml_doc(data_pathname);
266  dmz->build_thin_dmr(dmr);
267 
268  dmz->load_all_attributes(dmr);
269 #else
270  DmrppTypeFactory BaseFactory; // Use the factory for this handler's types
271  dmr->set_factory(&BaseFactory);
272 
273  DmrppParserSax2 parser;
274  ifstream in(data_pathname.c_str(), ios::in);
275  parser.intern(in, dmr);
276 
277  dmr->set_factory(0);
278 #endif
279 }
280 
294 {
295  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
296 
297  BESResponseObject *response = dhi.response_handler->get_response_object();
298  BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(response);
299  if (!bdmr) throw BESInternalError("Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
300 
301  try {
302  build_dmr_from_file(dhi.container, bdmr->get_dmr());
303  bdmr->set_dap4_constraint(dhi);
304  bdmr->set_dap4_function(dhi);
305  }
306  catch (...) {
307  handle_exception(__FILE__, __LINE__);
308  }
309 
310  BESDEBUG(MODULE, prolog << "END" << endl);
311 
312  return true;
313 }
314 
321 {
322  BESStopWatch sw;
323  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
324 
325  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
326 
327  BESResponseObject *response = dhi.response_handler->get_response_object();
328  auto *bdmr = dynamic_cast<BESDMRResponse *>(response);
329  if (!bdmr) throw BESInternalError("Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
330 
331  try {
332  build_dmr_from_file(dhi.container, bdmr->get_dmr());
333 
334  // We don't need all the attributes, so use the lazy-load feature implemented
335  // using overloads of the BaseType::set_send_p() method.
336 
337  bdmr->set_dap4_constraint(dhi);
338  bdmr->set_dap4_function(dhi);
339  }
340  catch (...) {
341  handle_exception(__FILE__, __LINE__);
342  }
343 
344  BESDEBUG(MODULE, prolog << "END" << endl);
345 
346  return true;
347 }
348 
355 template <class T>
356 void DmrppRequestHandler::get_dds_from_dmr_or_cache(BESDataHandlerInterface &dhi, T *bdds) {
357  string container_name_str = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
358 
359  DDS *dds = bdds->get_dds();
360  if (!container_name_str.empty()) dds->container_name(container_name_str);
361  string accessed = dhi.container->access();
362 
363  // Look in memory cache, if it's initialized
364  DDS *cached_dds_ptr = 0;
365  if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(accessed)))) {
366  BESDEBUG(MODULE, prolog << "DDS Cached hit for : " << accessed << endl);
367  *dds = *cached_dds_ptr;
368  }
369  else {
370  DMR dmr;
371  build_dmr_from_file(dhi.container, &dmr);
372 
373  delete dds; // delete the current one;
374  dds = dmr.getDDS(); // assign the new one.
375 
376  // Stuff it into the response.
377  bdds->set_dds(dds);
378 
379  // Cache it, if the cache is active.
380  if (dds_cache) {
381  dds_cache->add(new DDS(*dds), accessed);
382  }
383  }
384 }
385 
390 {
391  BESStopWatch sw;
392  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
393 
394  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
395 
396  BESResponseObject *response = dhi.response_handler->get_response_object();
397  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
398  if (!bdds) throw BESInternalError("Cast error, expected a BESDataDDSResponse object.", __FILE__, __LINE__);
399 
400  try {
401  get_dds_from_dmr_or_cache<BESDataDDSResponse>(dhi, bdds);
402  bdds->set_constraint(dhi);
403  bdds->clear_container();
404  }
405  catch (...) {
406  handle_exception(__FILE__, __LINE__);
407  }
408 
409  BESDEBUG(MODULE, prolog << "END" << endl);
410  return true;
411 }
412 
413 
418 {
419  BESStopWatch sw;
420  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
421 
422  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
423 
424  BESResponseObject *response = dhi.response_handler->get_response_object();
425  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *>(response);
426  if (!bdds) throw BESInternalError("Cast error, expected a BESDDSResponse object.", __FILE__, __LINE__);
427 
428  try {
429  get_dds_from_dmr_or_cache<BESDDSResponse>(dhi, bdds);
430 
431  bdds->set_constraint(dhi);
432  bdds->clear_container();
433  }
434  catch (...) {
435  handle_exception(__FILE__, __LINE__);
436  }
437 
438  BESDEBUG(MODULE, prolog << "END" << endl);
439  return true;
440 }
441 
447 {
448  BESStopWatch sw;
449  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
450 
451  BESResponseObject *response = dhi.response_handler->get_response_object();
452  BESDASResponse *bdas = dynamic_cast<BESDASResponse *>(response);
453  if (!bdas) throw BESInternalError("Cast error, expected a BESDASResponse object.", __FILE__, __LINE__);
454 
455  try {
456  string container_name_str = bdas->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
457 
458  DAS *das = bdas->get_das();
459  if (!container_name_str.empty()) das->container_name(container_name_str);
460  string accessed = dhi.container->access();
461 
462  // Look in memory cache (if it's initialized)
463  DAS *cached_das_ptr = 0;
464  if (das_cache && (cached_das_ptr = static_cast<DAS*>(das_cache->get(accessed)))) {
465  // copy the cached DAS into the BES response object
466  *das = *cached_das_ptr;
467  }
468  else {
469  DMR dmr;
470  build_dmr_from_file(dhi.container, &dmr);
471 
472  // Get a DDS from the DMR, getDDS() allocates all new objects. Use unique_ptr
473  // to ensure this is deleted. jhrg 11/12/21
474  // TODO Add a getDAS() method to DMR so we don't have to go the long way?
475  // Or not and drop the DAP2 stuff until the code is higher up the chain?
476  // jhrg 11/12/21
477  unique_ptr<DDS> dds(dmr.getDDS());
478 
479  // Load the BESDASResponse DAS from the DDS
480  dds->get_das(das);
481  Ancillary::read_ancillary_das(*das, accessed);
482 
483  // Add to cache if cache is active
484  if (das_cache) {
485  // copy because the BES deletes the DAS held by the DHI.
486  // TODO Change the DHI to use shared_ptr objects. I think ... jhrg 11/12/21
487  das_cache->add(new DAS(*das), accessed);
488  }
489  }
490 
491  bdas->clear_container();
492  }
493  catch (...) {
494  handle_exception(__FILE__, __LINE__);
495  }
496 
497  BESDEBUG(MODULE, prolog << "END" << endl);
498  return true;
499 }
500 
501 
502 bool DmrppRequestHandler::dap_build_vers(BESDataHandlerInterface &dhi)
503 {
504  BESVersionInfo *info = dynamic_cast<BESVersionInfo *>(dhi.response_handler->get_response_object());
505  if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
506 
507  info->add_module(MODULE_NAME, MODULE_VERSION);
508  return true;
509 }
510 
511 bool DmrppRequestHandler::dap_build_help(BESDataHandlerInterface &dhi)
512 {
513  BESInfo *info = dynamic_cast<BESInfo *>(dhi.response_handler->get_response_object());
514  if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
515 
516  // This is an example. If you had a help file you could load it like
517  // this and if your handler handled the following responses.
518  map<string, string> attrs;
519  attrs["name"] = MODULE_NAME;
520  attrs["version"] = MODULE_VERSION;
521  list<string> services;
522  BESServiceRegistry::TheRegistry()->services_handled(MODULE, services);
523  if (services.size() > 0) {
524  string handles = BESUtil::implode(services, ',');
525  attrs["handles"] = handles;
526  }
527  info->begin_tag("module", &attrs);
528  info->end_tag("module");
529 
530  return true;
531 }
532 
533 void DmrppRequestHandler::dump(ostream &strm) const
534 {
535  strm << BESIndent::LMarg << "DmrppRequestHandler::dump - (" << (void *) this << ")" << endl;
536  BESIndent::Indent();
538  BESIndent::UnIndent();
539 }
540 
541 } // namespace dmrpp
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Holds a DDS object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
informational response object
Definition: BESInfo.h:63
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual bool add_method(const std::string &name, p_request_handler_method method)
add a handler method to the request handler that knows how to fill in a specific response object
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:206
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:657
static CredentialsManager * theCM()
Returns the singleton instance of the CrednetialsManager.
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
static bool dap_build_dds(BESDataHandlerInterface &dhi)
void dump(std::ostream &strm) const override
dumps information about this object
static bool dap_build_dap2data(BESDataHandlerInterface &dhi)
static bool dap_build_dmr(BESDataHandlerInterface &dhi)
static bool dap_build_das(BESDataHandlerInterface &dhi)
static bool dap_build_dap4data(BESDataHandlerInterface &dhi)
Build a DAP4 data response. Adds timing to dap_build_dmr()