bes  Updated for version 3.20.10
BESDapResponseBuilder.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2011 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #include "config.h"
26 
27 #include <signal.h>
28 #include <unistd.h>
29 #include <sys/stat.h>
30 
31 #ifdef HAVE_UUID_UUID_H
32 #include <uuid/uuid.h> // used to build CID header value for data ddx
33 #elif defined(HAVE_UUID_H)
34 #include <uuid.h>
35 #else
36 #error "Could not find UUID library header"
37 #endif
38 
39 
40 #ifndef WIN32
41 #include <sys/wait.h>
42 #else
43 #include <io.h>
44 #include <fcntl.h>
45 #include <process.h>
46 #endif
47 
48 #include <iostream>
49 #include <string>
50 #include <sstream>
51 #include <fstream>
52 
53 #include <cstring>
54 #include <ctime>
55 
56 //#define DODS_DEBUG
57 #define CLEAR_LOCAL_DATA
58 #undef USE_LOCAL_TIMEOUT_SCHEME
59 
60 #include <libdap/DAS.h>
61 #include <libdap/DDS.h>
62 #include <libdap/Structure.h>
63 #include <libdap/ConstraintEvaluator.h>
64 #include <libdap/DDXParserSAX2.h>
65 #include <libdap/Ancillary.h>
66 #include <libdap/XDRStreamMarshaller.h>
67 #include <libdap/XDRFileUnMarshaller.h>
68 
69 #include <libdap/DMR.h>
70 #include <libdap/D4Group.h>
71 #include <libdap/D4Attributes.h>
72 #include <libdap/XMLWriter.h>
73 #include <libdap/D4AsyncUtil.h>
74 #include <libdap/D4StreamMarshaller.h>
75 #include <libdap/chunked_ostream.h>
76 #include <libdap/chunked_istream.h>
77 #include <libdap/D4ConstraintEvaluator.h>
78 #include <libdap/D4FunctionEvaluator.h>
79 #include <libdap/D4BaseTypeFactory.h>
80 
81 #include <libdap/ServerFunctionsList.h>
82 
83 #include <libdap/mime_util.h> // for last_modified_time() and rfc_822_date()
84 #include <libdap/escaping.h>
85 #include <libdap/util.h>
86 // #include <d4_function/D4FunctionEvaluator.h>
87 
88 #if USE_LOCAL_TIMEOUT_SCHEME
89 #ifndef WIN32
90 #include <libdap/SignalHandler.h>
91 #include <libdap/EventHandler.h>
92 #include <libdap/AlarmHandler.h>
93 #endif
94 #endif
95 
96 #include "TheBESKeys.h"
97 #include "BESDapResponseBuilder.h"
98 #include "BESContextManager.h"
99 #include "BESDapFunctionResponseCache.h"
100 #include "BESStoredDapResultCache.h"
101 
102 
103 #include "BESResponseObject.h"
104 #include "BESDDSResponse.h"
105 #include "BESDataDDSResponse.h"
106 #include "BESDMRResponse.h"
107 #include "BESDataHandlerInterface.h"
108 #include "BESInternalFatalError.h"
109 #include "BESSyntaxUserError.h"
110 #include "BESDataNames.h"
111 
112 #include "BESRequestHandler.h"
113 #include "BESRequestHandlerList.h"
114 #include "BESNotFoundError.h"
115 
116 #include "BESUtil.h"
117 #include "BESDebug.h"
118 #include "BESLog.h"
119 #include "BESStopWatch.h"
120 #include "DapFunctionUtils.h"
121 
122 using namespace std;
123 using namespace libdap;
124 
125 const string CRLF = "\r\n"; // Change here, expr-test.cc
126 const string BES_KEY_TIMEOUT_CANCEL = "BES.CancelTimeoutOnSend";
127 
128 #define MODULE "dap"
129 #define prolog std::string("BESDapResponseBuilder::").append(__func__).append("() - ")
130 
136 {
137  bool found = false;
138  string cancel_timeout_on_send = "";
139  TheBESKeys::TheKeys()->get_value(BES_KEY_TIMEOUT_CANCEL, cancel_timeout_on_send, found);
140  if (found && !cancel_timeout_on_send.empty()) {
141  // The default value is false.
142  downcase(cancel_timeout_on_send);
143  if (cancel_timeout_on_send == "yes" || cancel_timeout_on_send == "true")
144  d_cancel_timeout_on_send = true;
145  }
146 }
147 
148 BESDapResponseBuilder::~BESDapResponseBuilder()
149 {
150 #if USE_LOCAL_TIMEOUT_SCHEME
151  // If an alarm was registered, delete it. The register code in SignalHandler
152  // always deletes the old alarm handler object, so only the one returned by
153  // remove_handler needs to be deleted at this point.
154  delete dynamic_cast<AlarmHandler*>(SignalHandler::instance()->remove_handler(SIGALRM));
155 #endif
156 }
157 
165 {
166  return d_dap2ce;
167 }
168 
180 {
181  d_dap2ce = www2id(_ce, "%", "%20");
182 }
183 
188 {
189  return d_dap4ce;
190 }
191 
203 {
204  d_dap4ce = www2id(_ce, "%", "%20");
205 }
206 
211 {
212  return d_dap4function;
213 }
214 
227 {
228  d_dap4function = www2id(_func, "%", "%20");
229 }
230 
231 std::string BESDapResponseBuilder::get_store_result() const
232 {
233  return d_store_result;
234 }
235 
236 void BESDapResponseBuilder::set_store_result(std::string _sr)
237 {
238  d_store_result = _sr;
239  BESDEBUG(MODULE, prolog << "store_result: " << _sr << endl);
240 }
241 
242 std::string BESDapResponseBuilder::get_async_accepted() const
243 {
244  return d_async_accepted;
245 }
246 
247 void BESDapResponseBuilder::set_async_accepted(std::string _aa)
248 {
249  d_async_accepted = _aa;
250  BESDEBUG(MODULE, prolog << "set_async_accepted() - async_accepted: " << _aa << endl);
251 }
252 
262 {
263  return d_dataset;
264 }
265 
277 {
278  d_dataset = www2id(ds, "%", "%20");
279 }
280 
287 {
288  d_timeout = t;
289 }
290 
293 {
294  return d_timeout;
295 }
296 
303 void
305 {
306 #if USE_LOCAL_TIMEOUT_SCHEME
307 #ifndef WIN32
308  alarm(d_timeout);
309 #endif
310 #endif
311 }
312 
318 void
320 {
321 #if USE_LOCAL_TIMEOUT_SCHEME
322 #ifndef WIN32
323  alarm(0);
324 #endif
325 #endif
326 }
327 
343 {
344  if (d_cancel_timeout_on_send)
345  alarm(0);
346 }
347 
357 {
358 #if USE_LOCAL_TIMEOUT_SCHEME
359 #ifndef WIN32
360  SignalHandler *sh = SignalHandler::instance();
361  EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler());
362  delete old_eh;
363 #endif
364 #endif
365 }
366 
367 
374 {
375 #if USE_LOCAL_TIMEOUT_SCHEME
376 #ifndef WIN32
377  if (d_timeout > 0) {
378  SignalHandler *sh = SignalHandler::instance();
379  EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler());
380  delete old_eh;
381  alarm(d_timeout);
382  }
383 #endif
384 #endif
385 }
386 
401 static string::size_type find_closing_paren(const string &ce, string::size_type pos)
402 {
403  // Iterate over the string finding all ( or ) characters until the matching ) is found.
404  // For each ( found, increment count. When a ) is found and count is zero, it is the
405  // matching closing paren, otherwise, decrement count and keep looking.
406  int count = 1;
407  do {
408  pos = ce.find_first_of("()", pos + 1);
409  if (pos == string::npos){
410  stringstream msg;
411  msg << "Expected to find a matching closing parenthesis in: " << ce;
412  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
413  }
414 
415  if (ce[pos] == '(')
416  ++count;
417  else
418  --count; // must be ')'
419 
420  } while (count > 0);
421 
422  return pos;
423 }
424 
431 void BESDapResponseBuilder::split_ce(ConstraintEvaluator &eval, const string &expr)
432 {
433  BESDEBUG(MODULE, prolog << "source expression: " << expr << endl);
434 
435  string ce;
436  if (!expr.empty())
437  ce = expr;
438  else
439  ce = d_dap2ce;
440 
441  string btp_function_ce = "";
442  string::size_type pos = 0;
443 
444  // This hack assumes that the functions are listed first. Look for the first
445  // open paren and the last closing paren to accommodate nested function calls
446  string::size_type first_paren = ce.find("(", pos);
447  string::size_type closing_paren = string::npos;
448  if (first_paren != string::npos) closing_paren = find_closing_paren(ce, first_paren); //ce.find(")", pos);
449 
450  while (first_paren != string::npos && closing_paren != string::npos) {
451  // Maybe a BTP function; get the name of the potential function
452  string name = ce.substr(pos, first_paren - pos);
453 
454  // is this a BTP function
455  btp_func f;
456  if (eval.find_function(name, &f)) {
457  // Found a BTP function
458  if (!btp_function_ce.empty()) btp_function_ce += ",";
459  btp_function_ce += ce.substr(pos, closing_paren + 1 - pos);
460  ce.erase(pos, closing_paren + 1 - pos);
461  if (ce[pos] == ',') ce.erase(pos, 1);
462  }
463  else {
464  pos = closing_paren + 1;
465  // exception?
466  if (pos < ce.length() && ce.at(pos) == ',') ++pos;
467  }
468 
469  first_paren = ce.find("(", pos);
470  closing_paren = ce.find(")", pos);
471  }
472 
473  d_dap2ce = ce;
474  d_btp_func_ce = btp_function_ce;
475 
476  BESDEBUG(MODULE, prolog << "Modified constraint: " << d_dap2ce << endl);
477  BESDEBUG(MODULE, prolog << "BTP Function part: " << btp_function_ce << endl);
478  BESDEBUG(MODULE, prolog << "END" << endl);
479 }
480 
487 static void
488 throw_if_dap2_response_too_big(DDS *dds)
489 {
490  if (dds->too_big()) {
491 #if 0
492  stringstream msg;
493  msg << "The Request for " << request_size / 1024 << " kilobytes is too large; ";
494  msg << "requests on this server are limited to "
495  + long_to_string(dds->get_response_limit() /1024) + "KB.";
496  throw Error(msg.str());
497 #endif
498  stringstream msg;
499  msg << "The submitted DAP2 request will generate a " << dds->get_request_size_kb(true);
500  msg << " kilobyte response, which is too large. ";
501  msg << "The maximum response size for this server is limited to " << dds->get_response_limit_kb();
502  msg << " kilobytes.";
503  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
504  }
505 }
506 
507 static void
508 throw_if_dap4_response_too_big(DMR &dmr)
509 {
510  if (dmr.too_big()) {
511  stringstream msg;
512  msg << "The submitted DAP4 request will generate a " << dmr.request_size_kb(true);
513  msg << " kilobyte response, which is too large. ";
514  msg << "The maximum response size for this server is limited to " << dmr.response_limit_kb();
515  msg << " kilobytes.";
516  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
517  }
518 }
519 
534 void BESDapResponseBuilder::send_das(ostream &out, DAS &das, bool with_mime_headers) const
535 {
536  if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
537 
538  das.print(out);
539 
540  out << flush;
541 }
542 
560 void BESDapResponseBuilder::send_das(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool constrained,
561  bool with_mime_headers)
562 {
563 #if USE_LOCAL_TIMEOUT_SCHEME
564  // Set up the alarm.
565  establish_timeout(out);
566  dds.set_timeout(d_timeout);
567 #endif
568  if (!constrained) {
569  if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
570 
571  conditional_timeout_cancel();
572 
573  (*dds)->print_das(out);
574  out << flush;
575 
576  return;
577  }
578 
579  split_ce(eval);
580 
581  // If there are functions, parse them and eval.
582  // Use that DDS and parse the non-function ce
583  // Serialize using the second ce and the second dds
584  if (!d_btp_func_ce.empty()) {
585  ConstraintEvaluator func_eval;
586  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
587 
588  DDS *fdds = 0; // nulll_ptr
589  if (responseCache && responseCache->can_be_cached(*dds, get_btp_func_ce())) {
590  fdds = responseCache->get_or_cache_dataset(*dds, get_btp_func_ce());
591  }
592  else {
593  func_eval.parse_constraint(get_btp_func_ce(), **dds);
594  fdds = func_eval.eval_function_clauses(**dds);
595  }
596 
597  delete *dds; *dds = 0;
598  *dds = fdds;
599 
600  if (with_mime_headers)
601  set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
602 
603  conditional_timeout_cancel();
604 
605  (*dds)->print_das(out);
606  }
607  else {
608  eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
609 
610  if (with_mime_headers)
611  set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
612 
613  conditional_timeout_cancel();
614 
615  (*dds)->print_das(out);
616  }
617 
618  out << flush;
619 }
620 
621 
640 void BESDapResponseBuilder::send_dds(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool constrained,
641  bool with_mime_headers)
642 {
643  if (!constrained) {
644  if (with_mime_headers)
645  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
646 
647  conditional_timeout_cancel();
648 
649  (*dds)->print(out);
650  out << flush;
651  return;
652  }
653 
654 #if USE_LOCAL_TIMEOUT_SCHEME
655  // Set up the alarm.
656  establish_timeout(out);
657  dds.set_timeout(d_timeout);
658 #endif
659 
660  // Split constraint into two halves
661  split_ce(eval);
662 
663  // If there are functions, parse them and eval.
664  // Use that DDS and parse the non-function ce
665  // Serialize using the second ce and the second dds
666  if (!d_btp_func_ce.empty()) {
667  ConstraintEvaluator func_eval;
668 
669  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
670 
671  DDS *fdds = 0; // nulll_ptr
672  if (responseCache && responseCache->can_be_cached(*dds, get_btp_func_ce())) {
673  fdds = responseCache->get_or_cache_dataset(*dds, get_btp_func_ce());
674  }
675  else {
676  func_eval.parse_constraint(get_btp_func_ce(), **dds);
677  fdds = func_eval.eval_function_clauses(**dds);
678  }
679 
680  delete *dds; *dds = 0;
681  *dds = fdds;
682 
683  // Server functions might mark variables to use their read()
684  // methods. Clear that so the CE in d_dap2ce will control what is
685  // sent. If that is empty (there was only a function call) all
686  // of the variables in the intermediate DDS (i.e., the function
687  // result) will be sent.
688  (*dds)->mark_all(false);
689 
690  // This next step utilizes a well known static method (so really it's a function;),
691  // promote_function_output_structures() to look for
692  // one or more top level Structures whose name indicates (by way of ending with
693  // "_unwrap") that their contents should be promoted (aka moved) to the top level.
694  // This is in support of a hack around the current API where server side functions
695  // may only return a single DAP object and not a collection of objects. The name suffix
696  // "_unwrap" is used as a signal from the function to the the various response
697  // builders and transmitters that the representation needs to be altered before
698  // transmission, and that in fact is what happens in our friend
699  // promote_function_output_structures()
700  promote_function_output_structures(*dds);
701 
702  eval.parse_constraint(d_dap2ce, **dds);
703 
704  if (with_mime_headers)
705  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
706 
707 
708  conditional_timeout_cancel();
709 
710  (*dds)->print_constrained(out);
711  }
712  else {
713  eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
714 
715  if (with_mime_headers)
716  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset),(*dds)->get_dap_version());
717 
718  conditional_timeout_cancel();
719 
720  (*dds)->print_constrained(out);
721  }
722 
723  out << flush;
724 }
725 
726 #ifdef DAP2_STORED_RESULTS
741 bool BESDapResponseBuilder::store_dap2_result(ostream &out, DDS &dds, ConstraintEvaluator &eval)
742 {
743  if (get_store_result().empty()) return false;
744 
745  string serviceUrl = get_store_result();
746 
747  XMLWriter xmlWrtr;
748  D4AsyncUtil d4au;
749 
750  // FIXME Keys should be read in initialize(). Also, I think the D4AsyncUtil should
751  // be removed from libdap - it is much more about how the BES processes these kinds
752  // of operations. Change this when working on the response caching for ODSIP. But...
753  // do we really need to put the style sheet in the bes.conf file? Should it be baked
754  // into the code (because we don't want people to change it)?
755  bool found;
756  string *stylesheet_ref = 0, ss_ref_value;
757  TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
758  if (found && ss_ref_value.length() > 0) {
759  stylesheet_ref = &ss_ref_value;
760  }
761 
763  if (resultCache == NULL) {
764 
770  string msg = "The Stored Result request cannot be serviced. ";
771  msg += "Unable to acquire StoredResultCache instance. ";
772  msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
773 
774  BESDEBUG(MODULE, prolog << "[WARNING] " << msg << endl);
775 
776  d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
777  out << xmlWrtr.get_doc();
778  out << flush;
779 
780  BESDEBUG(MODULE,prolog << "Sent AsyncRequestRejected" << endl);
781  }
782  else if (get_async_accepted().length() != 0) {
783 
787  BESDEBUG(MODULE, prolog << "serviceUrl="<< serviceUrl << endl);
788 
790  string storedResultId = "";
791  storedResultId = resultCache->store_dap2_result(dds, get_ce(), this, &eval);
792 
793  BESDEBUG(MODULE, prolog << "storedResultId='"<< storedResultId << "'" << endl);
794 
795  string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
796  BESDEBUG(MODULE, prolog << "targetURL='"<< targetURL << "'" << endl);
797 
798  XMLWriter xmlWrtr;
799  d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
800  out << xmlWrtr.get_doc();
801  out << flush;
802 
803  BESDEBUG(MODULE, prolog << "Sent DAP4 AsyncAccepted response" << endl);
804  }
805  else {
810  d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
811  out << xmlWrtr.get_doc();
812  out << flush;
813 
814  BESDEBUG(MODULE, prolog << "Sent DAP4 AsyncRequired response" << endl);
815  }
816 
817  return true;
818 
819 }
820 #endif
821 
825 void BESDapResponseBuilder::serialize_dap2_data_dds(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool ce_eval)
826 {
827  BESStopWatch sw;
828  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
829 
830  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
831 
832  (*dds)->print_constrained(out);
833  out << "Data:\n";
834  out << flush;
835 
836  XDRStreamMarshaller m(out);
837 
838  // This only has an effect when the timeout in BESInterface::execute_request()
839  // is set. Otherwise it does nothing.
840  conditional_timeout_cancel();
841 
842  // Send all variables in the current projection (send_p())
843  for (DDS::Vars_iter i = (*dds)->var_begin(); i != (*dds)->var_end(); i++) {
844  if ((*i)->send_p()) {
845  (*i)->serialize(eval, **dds, m, ce_eval);
846 #ifdef CLEAR_LOCAL_DATA
847  (*i)->clear_local_data();
848 #endif
849  }
850  }
851 
852  BESDEBUG(MODULE, prolog << "END" << endl);
853 }
854 
855 #ifdef DAP2_STORED_RESULTS
864 void BESDapResponseBuilder::serialize_dap2_data_ddx(ostream &out, DDS **dds, ConstraintEvaluator &eval,
865  const string &boundary, const string &start, bool ce_eval)
866 {
867  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
868 
869  // Write the MPM headers for the DDX (text/xml) part of the response
870  libdap::set_mime_ddx_boundary(out, boundary, start, dods_ddx, x_plain);
871 
872  // Make cid
873  uuid_t uu;
874  uuid_generate(uu);
875  char uuid[37];
876  uuid_unparse(uu, &uuid[0]);
877  char domain[256];
878  if (getdomainname(domain, 255) != 0 || strlen(domain) == 0) strncpy(domain, "opendap.org", 255);
879 
880  string cid = string(&uuid[0]) + "@" + string(&domain[0]);
881 
882  // Send constrained DDX with a data blob reference.
883  // Note: CID passed but ignored jhrg 10/20/15
884  (*dds)->print_xml_writer(out, true, cid);
885 
886  // write the data part mime headers here
887  set_mime_data_boundary(out, boundary, cid, dods_data_ddx /* old value dap4_data*/, x_plain);
888 
889  XDRStreamMarshaller m(out);
890 
891  conditional_timeout_cancel();
892 
893 
894  // Send all variables in the current projection (send_p()).
895  for (DDS::Vars_iter i = (*dds)->var_begin(); i != (*dds)->var_end(); i++) {
896  if ((*i)->send_p()) {
897  (*i)->serialize(eval, **dds, m, ce_eval);
898 #ifdef CLEAR_LOCAL_DATA
899  (*i)->clear_local_data();
900 #endif
901  }
902  }
903 
904  BESDEBUG(MODULE, prolog << "END" << endl);
905 }
906 #endif
907 
925 {
926 #if USE_LOCAL_TIMEOUT_SCHEME
927  alarm(0);
928 #endif
929 }
930 
944 libdap::DDS *
946 {
947  BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
948 
949  dhi.first_container();
950 
951  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *>(obj);
952  if (!bdds) throw BESInternalFatalError("Expected a BESDDSResponse instance", __FILE__, __LINE__);
953 
954  DDS *dds = bdds->get_dds();
955 
956  set_dataset_name(dds->filename());
957  set_ce(dhi.data[POST_CONSTRAINT]);
958  set_async_accepted(dhi.data[ASYNC]);
959  set_store_result(dhi.data[STORE_RESULT]);
960 
961  ConstraintEvaluator &eval = bdds->get_ce();
962 
963  // Split constraint into two halves
964  split_ce(eval);
965 
966  // If there are functions, parse them and eval.
967  // Use that DDS and parse the non-function ce
968  // Serialize using the second ce and the second dds
969  if (!d_btp_func_ce.empty()) {
970  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
971 
972  ConstraintEvaluator func_eval;
973  DDS *fdds = 0; // nulll_ptr
974  if (responseCache && responseCache->can_be_cached(dds, get_btp_func_ce())) {
975  fdds = responseCache->get_or_cache_dataset(dds, get_btp_func_ce());
976  }
977  else {
978  func_eval.parse_constraint(get_btp_func_ce(), *dds);
979  fdds = func_eval.eval_function_clauses(*dds);
980  }
981 
982  delete dds; // Delete so that we can ...
983  bdds->set_dds(fdds); // Transfer management responsibility
984  dds = fdds;
985 
986  dds->mark_all(false);
987 
988  promote_function_output_structures(dds);
989  }
990 
991  eval.parse_constraint(d_dap2ce, *dds); // Throws Error if the ce doesn't parse.
992  BESDEBUG(MODULE, prolog << "END"<< endl);
993 
994  return dds;
995 }
996 
1014 libdap::DDS *
1016 {
1017  BESStopWatch sw;
1018  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1019 
1020  BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
1021 
1022  dhi.first_container();
1023 
1024  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(obj);
1025  if (!bdds) throw BESInternalFatalError("Expected a BESDataDDSResponse instance", __FILE__, __LINE__);
1026 
1027  DDS *dds = bdds->get_dds();
1028 
1029  set_dataset_name(dds->filename());
1030  set_ce(dhi.data[POST_CONSTRAINT]);
1031  set_async_accepted(dhi.data[ASYNC]);
1032  set_store_result(dhi.data[STORE_RESULT]);
1033 
1034 
1035  // This function is used by all fileout modules and they need to include the attributes in data access.
1036  // So obtain the attributes if necessary. KY 2019-10-30
1037  if(bdds->get_ia_flag() == false) {
1038  BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(dhi.container->get_container_type());
1039  besRH->add_attributes(dhi);
1040  }
1041 
1042  ConstraintEvaluator &eval = bdds->get_ce();
1043 
1044  // Split constraint into two halves; stores the function and non-function parts in this instance.
1045  split_ce(eval);
1046 
1047  // If there are functions, parse them and eval.
1048  // Use that DDS and parse the non-function ce
1049  // Serialize using the second ce and the second dds
1050  if (!get_btp_func_ce().empty()) {
1051  BESDEBUG(MODULE,prolog << "Found function(s) in CE: " << get_btp_func_ce() << endl);
1052 
1053  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
1054 
1055  ConstraintEvaluator func_eval;
1056  DDS *fdds = nullptr;
1057  if (responseCache && responseCache->can_be_cached(dds, get_btp_func_ce())) {
1058  fdds = responseCache->get_or_cache_dataset(dds, get_btp_func_ce());
1059  }
1060  else {
1061  func_eval.parse_constraint(get_btp_func_ce(), *dds);
1062  fdds = func_eval.eval_function_clauses(*dds);
1063  }
1064 
1065  delete dds; // Delete so that we can ...
1066  bdds->set_dds(fdds); // Transfer management responsibility
1067  dds = fdds;
1068 
1069  // Server functions might mark (i.e. setting send_p) so variables will use their read()
1070  // methods. Clear that so the CE in d_dap2ce will control what is
1071  // sent. If that is empty (there was only a function call) all
1072  // of the variables in the intermediate DDS (i.e., the function
1073  // result) will be sent.
1074  dds->mark_all(false);
1075 
1076  // Look for one or more top level Structures whose name indicates (by way of ending with
1077  // "_uwrap") that their contents should be moved to the top level.
1078  //
1079  // This is in support of a hack around the current API where server side functions
1080  // may only return a single DAP object and not a collection of objects. The name suffix
1081  // "_unwrap" is used as a signal from the function to the the various response
1082  // builders and transmitters that the representation needs to be altered before
1083  // transmission, and that in fact is what happens in our friend
1084  // promote_function_output_structures()
1085  promote_function_output_structures(dds);
1086  }
1087 
1088  // evaluate the rest of the CE - the part that follows the function calls.
1089  eval.parse_constraint(get_ce(), *dds);
1090 
1091  dds->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1092 
1093  throw_if_dap2_response_too_big(dds);
1094 
1095  // Iterate through the variables in the DataDDS and read
1096  // in the data if the variable has the send flag set.
1097  for (DDS::Vars_iter i = dds->var_begin(), e = dds->var_end(); i != e; ++i) {
1098  if ((*i)->send_p()) {
1099  try {
1100  (*i)->intern_data(eval, *dds);
1101  }
1102  catch(std::exception &e) {
1103  throw BESSyntaxUserError(string("Caught a C++ standard exception while working on '") + (*i)->name() + "' The error was: " + e.what(), __FILE__, __LINE__);
1104  }
1105  }
1106  }
1107 
1108  BESDEBUG(MODULE, prolog << "END"<< endl);
1109 
1110  return dds;
1111 }
1112 
1113 
1126 void BESDapResponseBuilder::send_dap2_data(ostream &data_stream, DDS **dds, ConstraintEvaluator &eval,
1127  bool with_mime_headers)
1128 {
1129  BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
1130 
1131 #if USE_LOCAL_TIMEOUT_SCHEME
1132  // Set up the alarm.
1133  establish_timeout(data_stream);
1134  dds.set_timeout(d_timeout);
1135 #endif
1136 
1137  // Split constraint into two halves
1138  split_ce(eval);
1139 
1140  // If there are functions, parse them and eval.
1141  // Use that DDS and parse the non-function ce
1142  // Serialize using the second ce and the second dds
1143  if (!get_btp_func_ce().empty()) {
1144  BESDEBUG(MODULE,prolog << "Found function(s) in CE: " << get_btp_func_ce() << endl);
1145 
1146  BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1147 
1148  ConstraintEvaluator func_eval;
1149  DDS *fdds = 0; // nulll_ptr
1150  if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1151  fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1152  }
1153  else {
1154  func_eval.parse_constraint(get_btp_func_ce(), **dds);
1155  fdds = func_eval.eval_function_clauses(**dds);
1156  }
1157 
1158  delete *dds; *dds = 0;
1159  *dds = fdds;
1160 
1161  (*dds)->mark_all(false);
1162 
1163  promote_function_output_structures(*dds);
1164 
1165  // evaluate the rest of the CE - the part that follows the function calls.
1166  eval.parse_constraint(get_ce(), **dds);
1167 
1168  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1169 
1170  throw_if_dap2_response_too_big(*dds);
1171 
1172  if (with_mime_headers)
1173  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1174 
1175 #if STORE_DAP2_RESULT_FEATURE
1176  // This means: if we are not supposed to store the result, then serialize it.
1177  if (!store_dap2_result(data_stream, **dds, eval)) {
1178  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1179  }
1180 #else
1181  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1182 #endif
1183 
1184  }
1185  else {
1186  BESDEBUG(MODULE, prolog << "Simple constraint" << endl);
1187 
1188  eval.parse_constraint(get_ce(), **dds); // Throws Error if the ce doesn't parse.
1189 
1190  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1191 
1192  throw_if_dap2_response_too_big(*dds);
1193 
1194  if (with_mime_headers)
1195  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1196 
1197 #if STORE_DAP2_RESULT_FEATURE
1198  // This means: if we are not supposed to store the result, then serialize it.
1199  if (!store_dap2_result(data_stream, **dds, eval)) {
1200  serialize_dap2_data_dds(data_stream, dds, eval);
1201  }
1202 #else
1203  serialize_dap2_data_dds(data_stream, dds, eval);
1204 #endif
1205  }
1206 
1207  data_stream << flush;
1208 
1209  BESDEBUG(MODULE, prolog << "END"<< endl);
1210 
1211 }
1212 
1213 void BESDapResponseBuilder::send_dap2_data(BESDataHandlerInterface &dhi, DDS **dds, ConstraintEvaluator &eval,
1214  bool with_mime_headers)
1215 {
1216  BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
1217 
1218  ostream & data_stream = dhi.get_output_stream();
1219 #if USE_LOCAL_TIMEOUT_SCHEME
1220  // Set up the alarm.
1221  establish_timeout(data_stream);
1222  dds.set_timeout(d_timeout);
1223 #endif
1224 
1225  // Split constraint into two halves
1226  split_ce(eval);
1227 
1228  // If there are functions, parse them and eval.
1229  // Use that DDS and parse the non-function ce
1230  // Serialize using the second ce and the second dds
1231  if (!get_btp_func_ce().empty()) {
1232  BESDEBUG(MODULE, prolog << "Found function(s) in CE: " << get_btp_func_ce() << endl);
1233 
1234  // Server-side functions need to include the attributes in data access.
1235  // So obtain the attributes if necessary. KY 2019-10-30
1236  {
1237  BESResponseObject *response = dhi.response_handler->get_response_object();
1238  auto *bdds = dynamic_cast<BESDataDDSResponse *> (response);
1239  if (!bdds)
1240  throw BESInternalError("cast error", __FILE__, __LINE__);
1241 
1242  if(!bdds->get_ia_flag()) {
1243  BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(dhi.container->get_container_type());
1244  besRH->add_attributes(dhi);
1245  }
1246  }
1247 
1248  BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1249  ConstraintEvaluator func_eval;
1250  DDS *fdds = nullptr;
1251  if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1252  fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1253  }
1254  else {
1255  func_eval.parse_constraint(get_btp_func_ce(), **dds);
1256  fdds = func_eval.eval_function_clauses(**dds);
1257  }
1258 
1259  delete *dds;
1260  *dds = nullptr;
1261  *dds = fdds;
1262 
1263  (*dds)->mark_all(false);
1264 
1265  promote_function_output_structures(*dds);
1266 
1267  // evaluate the rest of the CE - the part that follows the function calls.
1268  eval.parse_constraint(get_ce(), **dds);
1269 
1270  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1271 
1272  throw_if_dap2_response_too_big(*dds);
1273 
1274  if (with_mime_headers)
1275  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1276 
1277 #if STORE_DAP2_RESULT_FEATURE
1278  // This means: if we are not supposed to store the result, then serialize it.
1279  if (!store_dap2_result(data_stream, **dds, eval)) {
1280  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1281  }
1282 #else
1283  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1284 #endif
1285 
1286  }
1287  else {
1288  BESDEBUG(MODULE, prolog << "Simple constraint" << endl);
1289 
1290  eval.parse_constraint(get_ce(), **dds); // Throws Error if the ce doesn't parse.
1291 
1292  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1293 
1294  throw_if_dap2_response_too_big(*dds);
1295 
1296  if (with_mime_headers)
1297  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1298 
1299 #if STORE_DAP2_RESULT_FEATURE
1300  // This means: if we are not supposed to store the result, then serialize it.
1301  if (!store_dap2_result(data_stream, **dds, eval)) {
1302  serialize_dap2_data_dds(data_stream, dds, eval);
1303  }
1304 #else
1305  serialize_dap2_data_dds(data_stream, dds, eval);
1306 #endif
1307  }
1308 
1309  data_stream << flush;
1310 
1311  BESDEBUG(MODULE, prolog << "END"<< endl);
1312 
1313 }
1327 void BESDapResponseBuilder::send_ddx(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool with_mime_headers)
1328 {
1329  if (d_dap2ce.empty()) {
1330  if (with_mime_headers)
1331  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1332 
1333  (*dds)->print_xml_writer(out, false /*constrained */, "");
1334  //dds.print(out);
1335  out << flush;
1336  return;
1337  }
1338 
1339 #if USE_LOCAL_TIMEOUT_SCHEME
1340  // Set up the alarm.
1341  establish_timeout(out);
1342  dds.set_timeout(d_timeout);
1343 #endif
1344 
1345  // Split constraint into two halves
1346  split_ce(eval);
1347 
1348  // If there are functions, parse them and eval.
1349  // Use that DDS and parse the non-function ce
1350  // Serialize using the second ce and the second dds
1351  if (!d_btp_func_ce.empty()) {
1352  BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1353 
1354  ConstraintEvaluator func_eval;
1355  DDS *fdds = 0; // nulll_ptr
1356  if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1357  fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1358  }
1359  else {
1360  func_eval.parse_constraint(get_btp_func_ce(), **dds);
1361  fdds = func_eval.eval_function_clauses(**dds);
1362  }
1363 
1364  delete *dds; *dds = 0;
1365  *dds = fdds;
1366 
1367  (*dds)->mark_all(false);
1368 
1369  promote_function_output_structures(*dds);
1370 
1371  eval.parse_constraint(d_dap2ce, **dds);
1372 
1373  if (with_mime_headers)
1374  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1375 
1376  conditional_timeout_cancel();
1377 
1378  (*dds)->print_xml_writer(out, true, "");
1379  }
1380  else {
1381  eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
1382 
1383  if (with_mime_headers)
1384  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1385 
1386  conditional_timeout_cancel();
1387 
1388 
1389  // dds.print_constrained(out);
1390  (*dds)->print_xml_writer(out, true, "");
1391  }
1392 
1393  out << flush;
1394 }
1395 
1396 void BESDapResponseBuilder::send_dmr(ostream &out, DMR &dmr, bool with_mime_headers)
1397 {
1398  // If the CE is not empty, parse it. The projections, etc., are set as a side effect.
1399  // If the parser returns false, the expression did not parse. The parser may also
1400  // throw Error
1401  if (!d_dap4ce.empty()) {
1402 
1403  BESDEBUG(MODULE, prolog << "Parsing DAP4 constraint: '"<< d_dap4ce << "'"<< endl);
1404 
1405  D4ConstraintEvaluator parser(&dmr);
1406  bool parse_ok = parser.parse(d_dap4ce);
1407  if (!parse_ok){
1408  stringstream msg;
1409  msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1410  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1411  }
1412  }
1413  // with an empty CE, send everything. Even though print_dap4() and serialize()
1414  // don't need this, other code may depend on send_p being set. This may change
1415  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1416  else {
1417  dmr.root()->set_send_p(true);
1418  }
1419 
1420  if (with_mime_headers) set_mime_text(out, dap4_dmr, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1421 
1422  conditional_timeout_cancel();
1423 
1424  BESDEBUG(MODULE, prolog << "dmr.request_xml_base(): '"<< dmr.request_xml_base() << "' (dmr: " << (void *) &dmr << ")" << endl);
1425 
1426  XMLWriter xml;
1427  dmr.print_dap4(xml, /*constrained &&*/!d_dap4ce.empty() /* true == constrained */);
1428  out << xml.get_doc() << flush;
1429 }
1430 
1431 void BESDapResponseBuilder::send_dap4_data_using_ce(ostream &out, DMR &dmr, bool with_mime_headers)
1432 {
1433  if (!d_dap4ce.empty()) {
1434  BESDEBUG(MODULE , "BESDapResponseBuilder::send_dap4_data_using_ce() - expression constraint is not empty. " <<endl);
1435  D4ConstraintEvaluator parser(&dmr);
1436  bool parse_ok = parser.parse(d_dap4ce);
1437  if (!parse_ok){
1438  stringstream msg;
1439  msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1440  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1441  }
1442  }
1443  // with an empty CE, send everything. Even though print_dap4() and serialize()
1444  // don't need this, other code may depend on send_p being set. This may change
1445  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1446  else {
1447  dmr.set_ce_empty(true);
1448  dmr.root()->set_send_p(true);
1449  }
1450 
1451  throw_if_dap4_response_too_big(dmr);
1452 
1453  // The following block is for debugging purpose. KY 05/13/2020
1454 #if !NDEBUG
1455  for (auto i = dmr.root()->var_begin(), e = dmr.root()->var_end(); i != e; ++i) {
1456  BESDEBUG(MODULE , prolog << (*i)->name() << endl);
1457  if ((*i)->send_p()) {
1458  BESDEBUG(MODULE , prolog << "Obtain data- " << (*i)->name() << endl);
1459  D4Attributes *d4_attrs = (*i)->attributes();
1460  BESDEBUG(MODULE , prolog << "Number of attributes " << d4_attrs << endl);
1461  for (auto ii = d4_attrs->attribute_begin(), ee = d4_attrs->attribute_end(); ii != ee; ++ii) {
1462  BESDEBUG(MODULE ,prolog << "Attribute name is " << (*ii)->name() << endl);
1463  }
1464  }
1465  }
1466 #endif
1467 
1468  if (!store_dap4_result(out, dmr)) {
1469  serialize_dap4_data(out, dmr, with_mime_headers);
1470  }
1471 }
1472 
1482 {
1483  BESStopWatch sw;
1484  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1485  if (!d_dap4ce.empty()) {
1486  BESDEBUG(MODULE , prolog << "Expression constraint is not empty. " <<endl);
1487  D4ConstraintEvaluator parser(&dmr);
1488  bool parse_ok = parser.parse(d_dap4ce);
1489  if (!parse_ok){
1490  stringstream msg;
1491  msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1492  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1493  }
1494  }
1495  // with an empty CE, send everything. Even though print_dap4() and serialize()
1496  // don't need this, other code may depend on send_p being set. This may change
1497  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1498  else {
1499  dmr.set_ce_empty(true);
1500  dmr.root()->set_send_p(true);
1501  }
1502  throw_if_dap4_response_too_big(dmr);
1503 }
1504 
1505 void BESDapResponseBuilder::send_dap4_data(ostream &out, DMR &dmr, bool with_mime_headers)
1506 {
1507  // If a function was passed in with this request, evaluate it and use that DMR
1508  // for the remainder of this request.
1509  // TODO Add caching for these function invocations
1510  if (!d_dap4function.empty()) {
1511  D4BaseTypeFactory d4_factory;
1512  DMR function_result(&d4_factory, "function_results");
1513 
1514  // Function modules load their functions onto this list. The list is
1515  // part of libdap, not the BES.
1516  if (!ServerFunctionsList::TheList()) {
1517  stringstream msg;
1518  msg << "The function expression could not be evaluated because ";
1519  msg << "there are no server-side functions defined on this server.";
1520  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1521  }
1522 
1523  D4FunctionEvaluator parser(&dmr, ServerFunctionsList::TheList());
1524  bool parse_ok = parser.parse(d_dap4function);
1525  if (!parse_ok){
1526  stringstream msg;
1527  msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1528  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1529  }
1530  parser.eval(&function_result);
1531 
1532  // Now use the results of running the functions for the remainder of the
1533  // send_data operation.
1534  send_dap4_data_using_ce(out, function_result, with_mime_headers);
1535  }
1536  else {
1537  send_dap4_data_using_ce(out, dmr, with_mime_headers);
1538  }
1539 }
1540 
1544 void BESDapResponseBuilder::serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers)
1545 {
1546  BESStopWatch sw;
1547  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1548 
1549  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
1550 
1551  if (with_mime_headers) set_mime_binary(out, dap4_data, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1552 
1553  BESDEBUG(MODULE, prolog << "dmr.request_xml_base(): \"" << dmr.request_xml_base() << "\""<< endl);
1554 
1555  // Write the DMR
1556  XMLWriter xml;
1557  dmr.print_dap4(xml, !d_dap4ce.empty());
1558 
1559  // now make the chunked output stream; set the size to be at least chunk_size
1560  // but make sure that the whole of the xml plus the CRLF can fit in the first
1561  // chunk. (+2 for the CRLF bytes).
1562  chunked_ostream cos(out, max((unsigned int) CHUNK_SIZE, xml.get_doc_size() + 2));
1563 
1564  conditional_timeout_cancel();
1565 
1566  // using flush means that the DMR and CRLF are in the first chunk.
1567  cos << xml.get_doc() << CRLF << flush;
1568 
1569  // Write the data, chunked with checksums
1570  D4StreamMarshaller m(cos);
1571  dmr.root()->serialize(m, dmr, !d_dap4ce.empty());
1572 #ifdef CLEAR_LOCAL_DATA
1573  dmr.root()->clear_local_data();
1574 #endif
1575  cos << flush;
1576 
1577  BESDEBUG(MODULE, prolog << "END" << endl);
1578 }
1579 
1594 bool BESDapResponseBuilder::store_dap4_result(ostream &out, libdap::DMR &dmr)
1595 {
1596  if (get_store_result().length() != 0) {
1597  string serviceUrl = get_store_result();
1598 
1599  D4AsyncUtil d4au;
1600  XMLWriter xmlWrtr;
1601 
1602  // FIXME See above comment for store dap2 result
1603  bool found;
1604  string *stylesheet_ref = 0, ss_ref_value;
1605  TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
1606  if (found && ss_ref_value.length() > 0) {
1607  stylesheet_ref = &ss_ref_value;
1608  }
1609 
1611  if (resultCache == NULL) {
1612 
1618  string msg = "The Stored Result request cannot be serviced. ";
1619  msg += "Unable to acquire StoredResultCache instance. ";
1620  msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
1621 
1622  BESDEBUG(MODULE, prolog << "[WARNING] " << msg << endl);
1623  d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
1624  out << xmlWrtr.get_doc();
1625  out << flush;
1626  BESDEBUG(MODULE, prolog << "Sent AsyncRequestRejected" << endl);
1627 
1628  return true;
1629  }
1630 
1631  if (get_async_accepted().length() != 0) {
1632 
1636  BESDEBUG(MODULE, prolog << "serviceUrl="<< serviceUrl << endl);
1637 
1638  string storedResultId = "";
1639  storedResultId = resultCache->store_dap4_result(dmr, get_ce(), this);
1640 
1641  BESDEBUG(MODULE,prolog << "storedResultId='"<< storedResultId << "'" << endl);
1642 
1643  string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
1644  BESDEBUG(MODULE, prolog << "targetURL='"<< targetURL << "'" << endl);
1645 
1646  d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
1647  out << xmlWrtr.get_doc();
1648  out << flush;
1649  BESDEBUG(MODULE, prolog << "Sent AsyncAccepted" << endl);
1650 
1651  }
1652  else {
1657  d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
1658  out << xmlWrtr.get_doc();
1659  out << flush;
1660  BESDEBUG(MODULE, prolog << "Sent AsyncAccepted" << endl);
1661  }
1662 
1663  return true;
1664  }
1665 
1666  return false;
1667 }
1668 
1687 libdap::DMR *
1689 {
1690  BESStopWatch sw;
1691  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1692  BESDEBUG(MODULE , prolog << "BEGIN" << endl);
1693 
1694  unique_ptr<DMR> dmr = setup_dap4_intern_data(obj, dhi);
1695 
1696  intern_dap4_data_grp(dmr->root());
1697 
1698  return dmr.release();
1699 }
1700 
1701 unique_ptr<DMR>
1702 BESDapResponseBuilder::setup_dap4_intern_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
1703 {
1704  dhi.first_container();
1705 
1706  BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(obj);
1707  if (!bdmr) throw BESInternalFatalError("Expected a BESDMRResponse instance", __FILE__, __LINE__);
1708 
1709  unique_ptr<DMR> dmr(bdmr->get_dmr());
1710  // TL;DR Set the DMR managed by the BESResponseObject to nullptr to avoid calling ~DMR
1711  // twice on the same object.
1712  bdmr->set_dmr(nullptr);
1713  // Why this is here: In the past we designed the BESResponseObject class hierarchy to
1714  // manage the response object, which effectively means delete it when the BES is done
1715  // processing the request. We pass nullptr to set_dmr so that the BESResponseObject
1716  // does not call ~DMR since unique_ptr<> will do that for us.
1717 
1718  // Set the correct context by following intern_dap2_data()
1719  set_dataset_name(dmr->filename());
1720  set_dap4ce(dhi.data[DAP4_CONSTRAINT]);
1721  set_dap4function(dhi.data[DAP4_FUNCTION]);
1722  set_async_accepted(dhi.data[ASYNC]);
1723  set_store_result(dhi.data[STORE_RESULT]);
1724 
1725  if (!d_dap4function.empty()) {
1726  D4BaseTypeFactory d4_factory;
1727  unique_ptr<DMR> function_result(new DMR(&d4_factory, "function_results"));
1728 
1729  // Function modules load their functions onto this list. The list is
1730  // part of libdap, not the BES.
1731  if (!ServerFunctionsList::TheList()) {
1732  stringstream msg;
1733  msg << "The function expression could not be evaluated because ";
1734  msg << "there are no server-side functions defined on this server.";
1735  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1736  }
1737 
1738  D4FunctionEvaluator parser(dmr.get(), ServerFunctionsList::TheList());
1739  bool parse_ok = parser.parse(d_dap4function);
1740  if (!parse_ok){
1741  stringstream msg;
1742  msg << "Failed to parse the provided DAP4 server-side function expression: " << d_dap4function;
1743  throw BESSyntaxUserError(msg.str(),__FILE__,__LINE__);
1744  }
1745 
1746  parser.eval(function_result.get());
1747 
1748  // Now use the results of running the functions for the remainder of the
1749  // send_data operation.
1750  dap4_process_ce_for_intern_data(*function_result);
1751 
1752  return function_result;
1753  }
1754  else {
1755  BESDEBUG(MODULE , prolog << "Processing the constraint expression. " << endl);
1756  dap4_process_ce_for_intern_data(*dmr);
1757  return dmr;
1758  }
1759 }
1760 
1761 void BESDapResponseBuilder::intern_dap4_data_grp(libdap::D4Group* grp) {
1762  for (D4Group::Vars_iter i = grp->var_begin(), e = grp->var_end(); i != e; ++i) {
1763  BESDEBUG(MODULE , "BESDapResponseBuilder::intern_dap4_data() - "<< (*i)->name() <<endl);
1764  if ((*i)->send_p()) {
1765  BESDEBUG(MODULE , "BESDapResponseBuilder::intern_dap4_data() Obtain data- "<< (*i)->name() <<endl);
1766  (*i)->intern_data();
1767  }
1768  }
1769 
1770  for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1771  BESDEBUG(MODULE , "BESDapResponseBuilder::intern_dap4_data() group- "<< (*gi)->name() <<endl);
1772  intern_dap4_data_grp(*gi);
1773  }
1774 }
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
Holds a DDS object within the BES.
void set_dds(libdap::DDS *ddsIn)
libdap::ConstraintEvaluator & get_ce()
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
Cache the results from server functions.
virtual libdap::DDS * get_or_cache_dataset(libdap::DDS *dds, const std::string &constraint)
Return a DDS loaded with data that can be serialized back to a client.
virtual void set_dataset_name(const std::string _dataset)
Set the dataset pathname.
virtual std::string get_dataset_name() const
Get the dataset name.
virtual libdap::DMR * intern_dap4_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
virtual std::string get_dap4function() const
Get the DAP4 server side function expression.
virtual void split_ce(libdap::ConstraintEvaluator &eval, const std::string &expr="")
virtual std::string get_ce() const
Get the constraint expression.
virtual void set_dap4ce(std::string _ce)
virtual void remove_timeout() const
Transmit data.
virtual libdap::DDS * process_dap2_dds(BESResponseObject *obj, BESDataHandlerInterface &dhi)
Process a DDS (i.e., apply a constraint) for a non-DAP transmitter.
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)
virtual libdap::DDS * intern_dap2_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
virtual void send_dds(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool constrained=false, bool with_mime_headers=true)
Transmit a DDS.
virtual std::string get_dap4ce() const
Get the DAP4 constraint expression.
virtual void dap4_process_ce_for_intern_data(libdap::DMR &dmr)
Parse the DAP4 CE and throw if the request is too large.
virtual void establish_timeout(std::ostream &stream) const
virtual void send_ddx(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool with_mime_headers=true)
virtual void serialize_dap2_data_dds(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool ce_eval=true)
virtual void set_dap4function(std::string _func)
virtual bool store_dap4_result(std::ostream &out, libdap::DMR &dmr)
void set_timeout(int timeout=0)
virtual void set_ce(std::string _ce)
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
void first_container()
set the container pointer to the first container in the containers list
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
virtual BESRequestHandler * find_handler(const std::string &handler_name)
find and return the specified request handler
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
static BESStoredDapResultCache * get_instance()
error thrown if there is a user syntax error in the request or any other user error
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:840
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71