bes  Updated for version 3.20.10
FONcTransmitter.cc
1 // FONcTransmitter.cc
2 
3 // This file is part of BES Netcdf File Out Module
4 
5 // Copyright (c) 2004,2005 University Corporation for Atmospheric Research
6 // Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact University Corporation for Atmospheric Research at
23 // 3080 Center Green Drive, Boulder, CO 80301
24 
25 // (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
26 // Please read the full copyright statement in the file COPYRIGHT_UCAR.
27 //
28 // Authors:
29 // pwest Patrick West <pwest@ucar.edu>
30 // jgarcia Jose Garcia <jgarcia@ucar.edu>
31 // kyang Kent Yang <myang6@hdfgroup.org> (for DAP4/netCDF-4 enhancement)
32 
33 #include "config.h"
34 
35 
36 #ifdef HAVE_UNISTD_H
37 #include <unistd.h>
38 #endif
39 
40 #include <exception>
41 #include <sstream> // std::stringstream
42 #include <thread>
43 #include <future>
44 
45 #include <libdap/D4Group.h>
46 #include <libdap/D4Attributes.h>
47 #include <libdap/BaseType.h>
48 #include <libdap/escaping.h>
49 
50 #include <BESContextManager.h>
51 #include <BESDataDDSResponse.h>
52 #include <BESDapNames.h>
53 #include <BESDataNames.h>
54 #include <BESDebug.h>
55 #include <BESUtil.h>
56 #include <TempFile.h>
57 
58 
59 #include <BESLog.h>
60 #include <BESError.h>
61 #include <BESDapError.h>
62 #include <stringbuffer.h>
63 
64 #include "FONcBaseType.h"
65 #include "FONcRequestHandler.h"
66 #include "FONcTransmitter.h"
67 #include "FONcTransform.h"
68 
69 using namespace libdap;
70 using namespace std;
71 using namespace rapidjson;
72 
73 #define MODULE "fonc"
74 #define prolog string("FONcTransmitter::").append(__func__).append("() - ")
75 
76 
77 #if 0 // Moved to BESUtil.cc
78 // size of the buffer used to read from the temporary file built on disk and
79 // send data to the client over the network connection (socket/stream)
80 // #define OUTPUT_FILE_BLOCK_SIZE 4096
81 #endif
82 
96 {
97  add_method(DATA_SERVICE, FONcTransmitter::send_dap2_data);
98  add_method(DAP4DATA_SERVICE, FONcTransmitter::send_dap4_data);
99 }
100 
101 
119 {
120  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
121 
122  try { // Expanded try block so all DAP errors are caught. ndp 12/23/2015
123 
124  // This object closes the file when it goes out of scope.
125  bes::TempFile temp_file(FONcRequestHandler::temp_dir + "/ncXXXXXX");
126 
127  BESDEBUG(MODULE, prolog << "Building response file " << temp_file.get_name() << endl);
128 
129  ostream &strm = dhi.get_output_stream();
130  if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);
131 
132  BESDEBUG(MODULE, prolog << "Transmitting temp file " << temp_file.get_name() << endl);
133 
134  // Note that 'RETURN_CMD' is the same as the string that determines the file type:
135  // netcdf 3 or netcdf 4. Hack. jhrg 9/7/16
136  FONcTransform ft(obj, &dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
137 
138 #if 0
139  // This is used to signal the BESUtil::file_to_stream_task() this code is done
140  // writing to the file. WIP jhrg 6/4/21
141  atomic<bool> file_write_done(false);
142 
143  // Calling the 'packaged_task' here blocks, but we could have run the task in a thread.
144  // See: https://stackoverflow.com/questions/18143661/what-is-the-difference-between-packaged-task-and-async
145  // jhrg 6/4/21
146 
147  std::packaged_task<uint64_t(const string &, atomic<bool>&, ostream&)> task(BESUtil::file_to_stream_task);
148  std::future<uint64_t> result = task.get_future();
149  task(temp_file.get_name(), file_write_done, strm);
150 #endif
151 
152 #define TOGGLE_TASK 0
153  // TOGGLE_TASK 1 besstandalone -c bes.nc4.conf -i mem-pressure-tests/bescmd.xml > tmp2.nc4
154  // 151.13s user 8.75s system 98% cpu 2:41.69 total
155  // TOGGLE_TASK 0 besstandalone -c bes.nc4.conf -i mem-pressure-tests/bescmd.xml > tmp2.nc4
156  // 154.71s user 8.99s system 99% cpu 2:45.27 total
157  // TOGGLE_TASK 0 as above, but using BESUtil::file_to_stream(temp_file.get_name(), strm);
158  // and not ESUtil::file_to_stream_task(temp_file.get_name(), file_write_done, strm);
159  // 148.61s user 7.54s system 99% cpu 2:36.35 total
160 #if TOGGLE_TASK
161  // This code works without the sleep(1) hack in BESUtil::file_to_stream_task().
162  // Because it is marked as deferred, the task does not start until the future's
163  // get() method is run, after transform() has written all the data. jhrg 6/4/21
164  future<uint64_t> result = async(launch::deferred, &BESUtil::file_to_stream_task, temp_file.get_name(),
165  std::ref(file_write_done), std::ref(strm));
166 #endif
167  ft.transform_dap2(strm);
168 
169 #if 0
170  file_write_done = true;
171  uint64_t tcount = result.get();
172 #endif
173 
174  //original call before the 'task' hack was added:
175  //BESUtil::file_to_stream(temp_file.get_name(),strm);
176  //jhrg 6/4/21
177 
178 #if 0
179  // The task can be called like this right here
180  uint64_t tcount = BESUtil::file_to_stream_task(temp_file.get_name(), file_write_done, strm);
181 
182  // Or it can be run like this...
183  std::packaged_task<uint64_t(const string &, atomic<bool>&, ostream&)> task(BESUtil::file_to_stream_task);
184  std::future<uint64_t> result = task.get_future();
185  task(temp_file.get_name(), file_write_done, strm);
186  uint64_t tcount = result.get();
187 #endif
188  //BESDEBUG(MODULE, prolog << "NetCDF file bytes written " << tcount << endl);
189  }
190  catch (Error &e) {
191  throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
192  }
193  catch (BESError &e) {
194  throw;
195  }
196  catch (std::exception &e) {
197  throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
198  }
199  catch (...) {
200  throw BESInternalError("Failed to get read data: Unknown exception caught", __FILE__, __LINE__);
201  }
202 
203  BESDEBUG(MODULE, prolog << "END Transmitted as netcdf" << endl);
204 }
205 
224 {
225  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
226 
227  try { // Expanded try block so all DAP errors are caught. ndp 12/23/2015
228 
229  // This object closes the file when it goes out of scope.
230  bes::TempFile temp_file(FONcRequestHandler::temp_dir + "/ncXXXXXX");
231 
232  BESDEBUG(MODULE, prolog << "Building response file " << temp_file.get_name() << endl);
233  // Note that 'RETURN_CMD' is the same as the string that determines the file type:
234  // netcdf 3 or netcdf 4. Hack. jhrg 9/7/16
235  // FONcTransform ft(loaded_dmr, dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
236  FONcTransform ft(obj, &dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
237 
238  // Call the transform function for DAP4.
239  ft.transform_dap4();
240 
241  ostream &strm = dhi.get_output_stream();
242 
243 #if !NDEBUG
244  stringstream msg;
245  msg << prolog << "Using ostream: " << (void *) &strm << endl;
246  BESDEBUG(MODULE, msg.str());
247  INFO_LOG( msg.str());
248 #endif
249 
250  if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);
251 
252  BESDEBUG(MODULE, prolog << "Transmitting temp file " << temp_file.get_name() << endl);
253 
254  // FONcTransmitter::write_temp_file_to_stream(temp_file.get_fd(), strm); //, loaded_dds->filename(), ncVersion);
255  BESUtil::file_to_stream(temp_file.get_name(),strm);
256  }
257  catch (Error &e) {
258  throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
259  }
260  catch (BESError &e) {
261  throw;
262  }
263  catch (std::exception &e) {
264  throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
265  }
266  catch (...) {
267  throw BESInternalError("Failed to get read data: Unknown exception caught", __FILE__, __LINE__);
268  }
269 
270  BESDEBUG(MODULE, prolog << "END Transmitted as netcdf" << endl);
271 }
272 
273 
274 
275 
276 
277 
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
exception thrown if internal error encountered
Abstract base class representing a specific set of information in response to a request to the BES.
static uint64_t file_to_stream_task(const std::string &file_name, std::atomic< bool > &file_write_done, std::ostream &o_strm)
Definition: BESUtil.cc:1399
static void file_to_stream(const std::string &file_name, std::ostream &o_strm)
Copies the contents of the file identified by file_name to the stream o_strm.
Definition: BESUtil.cc:1226
Transformation object that converts an OPeNDAP DataDDS to a netcdf file.
Definition: FONcTransform.h:61
virtual void transform_dap2(ostream &strm)
Transforms each of the variables of the DataDDS to the NetCDF file.
virtual void transform_dap4()
Transforms each of the variables of the DMR to the NetCDF file.
static void send_dap4_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
The static method registered to transmit OPeNDAP data objects as a netcdf file.
static void send_dap2_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
The static method registered to transmit OPeNDAP data objects as a netcdf file.
FONcTransmitter()
Construct the FONcTransmitter, adding it with name netcdf to be able to transmit a data response.
Get a new temporary file.
Definition: TempFile.h:46
std::string get_name() const
Definition: TempFile.h:70
main RapidJSON namespace