BitShares-Core  6.1.0
BitShares blockchain implementation and command-line interface software
elasticsearch.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 oxarbitrage, and contributors.
3  *
4  * The MIT License
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
25 
26 #include <boost/algorithm/string/join.hpp>
27 
28 #include <fc/io/json.hpp>
30 
31 static size_t curl_write_function(void *contents, size_t size, size_t nmemb, void *userp)
32 {
33  ((std::string*)userp)->append((char*)contents, size * nmemb);
34  return size * nmemb;
35 }
36 
37 namespace graphene { namespace utilities {
38 
39 static bool handle_bulk_response( uint16_t http_code, const std::string& curl_read_buffer )
40 {
42  {
43  // all good, but check errors in response
44  fc::variant j = fc::json::from_string(curl_read_buffer);
45  bool errors = j["errors"].as_bool();
46  if( errors )
47  {
48  elog( "ES returned 200 but with errors: ${e}", ("e", curl_read_buffer) );
49  return false;
50  }
51  return true;
52  }
53 
55  {
56  elog( "413 error: Request too large. Can be low disk space. ${e}", ("e", curl_read_buffer) );
57  }
58  else if( curl_wrapper::http_response_code::HTTP_401 == http_code )
59  {
60  elog( "401 error: Unauthorized. ${e}", ("e", curl_read_buffer) );
61  }
62  else
63  {
64  elog( "${code} error: ${e}", ("code", std::to_string(http_code)) ("e", curl_read_buffer) );
65  }
66  return false;
67 }
68 
69 std::vector<std::string> createBulk(const fc::mutable_variant_object& bulk_header, std::string&& data)
70 {
71  std::vector<std::string> bulk;
72  fc::mutable_variant_object final_bulk_header;
73  final_bulk_header["index"] = bulk_header;
74  bulk.push_back(fc::json::to_string(final_bulk_header));
75  bulk.emplace_back(std::move(data));
76 
77  return bulk;
78 }
79 
81 {
82  return ( http_response_code::HTTP_200 == code );
83 }
84 
85 CURL* curl_wrapper::init_curl()
86 {
87  CURL* curl = curl_easy_init();
88  if( curl )
89  {
90  curl_easy_setopt( curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2 );
91  return curl;
92  }
93  FC_THROW( "Unable to init cURL" );
94 }
95 
96 curl_slist* curl_wrapper::init_request_headers()
97 {
98  curl_slist* request_headers = curl_slist_append( NULL, "Content-Type: application/json" );
99  FC_ASSERT( request_headers, "Unable to init cURL request headers" );
100  return request_headers;
101 }
102 
104 {
105  curl_easy_setopt( curl.get(), CURLOPT_HTTPHEADER, request_headers.get() );
106  curl_easy_setopt( curl.get(), CURLOPT_USERAGENT, "bitshares-core/6.1" );
107 }
108 
109 void curl_wrapper::curl_deleter::operator()( CURL* p_curl ) const
110 {
111  if( p_curl )
112  curl_easy_cleanup( p_curl );
113 }
114 
115 void curl_wrapper::curl_slist_deleter::operator()( curl_slist* slist ) const
116 {
117  if( slist )
118  curl_slist_free_all( slist );
119 }
120 
122  const std::string& url,
123  const std::string& auth,
124  const std::string& query ) const
125 {
127 
128  // Note: the variable curl has a long lifetime, it only gets initialized once, then be used many times,
129  // thus we need to clear old data
130 
131  // Note: host and auth are always the same in the program, ideally we don't need to set them every time
132  curl_easy_setopt( curl.get(), CURLOPT_URL, url.c_str() );
133  if( !auth.empty() )
134  curl_easy_setopt( curl.get(), CURLOPT_USERPWD, auth.c_str() );
135 
136  // Empty for GET, POST or HEAD, non-empty for DELETE or PUT
137  static const std::vector<std::string> http_request_method_custom_str = {
138  "", // GET
139  "", // POST
140  "", // HEAD
141  "PUT",
142  "DELETE",
143  "PATCH",
144  "OPTIONS"
145  };
146  const auto& custom_request = http_request_method_custom_str[static_cast<size_t>(method)];
147  const auto* p_custom_request = custom_request.empty() ? NULL : custom_request.c_str();
148  curl_easy_setopt( curl.get(), CURLOPT_CUSTOMREQUEST, p_custom_request );
149 
152  {
153  curl_easy_setopt( curl.get(), CURLOPT_HTTPGET, false );
154  curl_easy_setopt( curl.get(), CURLOPT_POST, true );
155  curl_easy_setopt( curl.get(), CURLOPT_POSTFIELDS, query.c_str() );
156  }
157  else // GET or DELETE (only these are used in this file)
158  {
159  curl_easy_setopt( curl.get(), CURLOPT_POSTFIELDS, NULL );
160  curl_easy_setopt( curl.get(), CURLOPT_POST, false );
161  curl_easy_setopt( curl.get(), CURLOPT_HTTPGET, true );
162  }
163 
164  curl_easy_setopt( curl.get(), CURLOPT_WRITEFUNCTION, curl_write_function );
165  curl_easy_setopt( curl.get(), CURLOPT_WRITEDATA, (void *)(&resp.content) );
166  curl_easy_perform( curl.get() );
167 
168  long code;
169  curl_easy_getinfo( curl.get(), CURLINFO_RESPONSE_CODE, &code );
170  resp.code = static_cast<uint16_t>( code );
171 
172  return resp;
173 }
174 
175 curl_wrapper::http_response curl_wrapper::get( const std::string& url, const std::string& auth ) const
176 {
177  return request( http_request_method::HTTP_GET, url, auth, "" );
178 }
179 
180 curl_wrapper::http_response curl_wrapper::del( const std::string& url, const std::string& auth ) const
181 {
182  return request( http_request_method::HTTP_DELETE, url, auth, "" );
183 }
184 
185 curl_wrapper::http_response curl_wrapper::post( const std::string& url, const std::string& auth,
186  const std::string& query ) const
187 {
188  return request( http_request_method::HTTP_POST, url, auth, query );
189 }
190 
191 curl_wrapper::http_response curl_wrapper::put( const std::string& url, const std::string& auth,
192  const std::string& query ) const
193 {
194  return request( http_request_method::HTTP_PUT, url, auth, query );
195 }
196 
198 {
199  const auto response = curl.get( base_url + "_nodes", auth );
200 
201  // Note: response.code is ignored here
202  return !response.content.empty();
203 }
204 
205 std::string es_client::get_version() const
206 { try {
207  const auto response = curl.get( base_url, auth );
208  if( !response.is_200() )
209  FC_THROW( "Error on es_client::get_version(): code = ${code}, message = ${message} ",
210  ("code", response.code) ("message", response.content) );
211 
212  fc::variant content = fc::json::from_string( response.content );
213  return content["version"]["number"].as_string();
215 
216 void es_client::check_version_7_or_above( bool& result ) const noexcept
217 {
218  static const int64_t version_7 = 7;
219  try {
220  const auto es_version = get_version();
221  auto dot_pos = es_version.find('.');
222  result = ( std::stoi(es_version.substr(0,dot_pos)) >= version_7 );
223  }
224  catch( ... )
225  {
226  wlog( "Unable to get ES version, assuming it is 7 or above" );
227  result = true;
228  }
229 }
230 
231 bool es_client::send_bulk( const std::vector<std::string>& bulk_lines ) const
232 {
233  auto bulk_str = boost::algorithm::join( bulk_lines, "\n" ) + "\n";
234  const auto response = curl.post( base_url + "_bulk", auth, bulk_str );
235 
236  return handle_bulk_response( response.code, response.content );
237 }
238 
239 bool es_client::del( const std::string& path ) const
240 {
241  const auto response = curl.del( base_url + path, auth );
242 
243  // Note: response.code is ignored here
244  return !response.content.empty();
245 }
246 
247 std::string es_client::get( const std::string& path ) const
248 {
249  const auto response = curl.get( base_url + path, auth );
250 
251  // Note: response.code is ignored here
252  return response.content;
253 }
254 
255 std::string es_client::query( const std::string& path, const std::string& query ) const
256 {
257  const auto response = curl.post( base_url + path, auth, query );
258 
259  // Note: response.code is ignored here
260  return response.content;
261 }
262 
263 fc::variant es_data_adaptor::adapt( const fc::variant_object& op, uint16_t max_depth )
264 {
265  if( 0 == max_depth )
266  {
267  fc::variant v;
269  return v;
270  }
271 
273 
274  // Note: these fields are maps, but were stored in ES as flattened arrays
275  static const std::unordered_set<std::string> flattened_fields = { "account_auths", "address_auths", "key_auths" };
276 
277  // Note:
278  // object arrays listed in this map are stored redundantly in ES, with one instance as a nested object and
279  // the other as a string for backward compatibility,
280  // object arrays not listed in this map are stored as nested objects only.
281  static const std::unordered_map<std::string, data_type> to_string_fields = {
282  { "parameters", data_type::array_type }, // in committee proposals, current_fees.parameters
283  { "op", data_type::static_variant_type }, // proposal_create_op.proposed_ops[*].op
284  { "proposed_ops", data_type::array_type },
285  { "operations", data_type::array_type }, // proposal_object.operations
286  { "initializer", data_type::static_variant_type },
287  { "policy", data_type::static_variant_type },
288  { "predicates", data_type::array_type },
289  { "active_special_authority", data_type::static_variant_type },
290  { "owner_special_authority", data_type::static_variant_type },
291  { "htlc_preimage_hash", data_type::static_variant_type },
292  { "argument", data_type::static_variant_type }, // for custom authority, restriction.argument
293  { "feeds", data_type::map_type }, // asset_bitasset_data_object.feeds
294  { "acceptable_collateral", data_type::map_type },
295  { "acceptable_borrowers", data_type::map_type }
296  };
297  std::vector<std::pair<std::string, fc::variants>> original_arrays;
298  std::vector<std::string> keys_to_rename;
299  for( auto& i : o )
300  {
301  const std::string& name = i.key();
302  auto& element = i.value();
303  if( element.is_object() )
304  {
305  const auto& vo = element.get_object();
306  if( vo.contains(name.c_str()) ) // transfer_operation.amount.amount
307  keys_to_rename.emplace_back(name);
308  element = adapt( vo, max_depth - 1 );
309  continue;
310  }
311 
312  if( !element.is_array() )
313  continue;
314 
315  auto& array = element.get_array();
316  if( to_string_fields.find(name) != to_string_fields.end() )
317  {
318  // make a backup (only if depth is sufficient) and convert to string
319  if( max_depth > 1 )
320  original_arrays.emplace_back( name, array );
321  element = fc::json::to_string(element);
322  }
323  else if( flattened_fields.find(name) != flattened_fields.end() )
324  {
325  // make a backup (only if depth is sufficient) and adapt the original
326  if( max_depth > 1 )
327  {
328  auto backup = array;
329  original_arrays.emplace_back( name, std::move( backup ) );
330  }
331  in_situ_adapt( array, max_depth - 1 );
332  }
333  else
334  in_situ_adapt( array, max_depth - 1 );
335  }
336 
337  for( const auto& i : keys_to_rename ) // transfer_operation.amount
338  {
339  std::string new_name = i + "_";
340  o[new_name] = fc::variant(o[i]);
341  o.erase(i);
342  }
343 
344  if( o.find("nonce") != o.end() )
345  {
346  o["nonce"] = o["nonce"].as_string();
347  }
348 
349  if( o.find("owner") != o.end() && o["owner"].is_string() ) // vesting_balance_*_operation.owner
350  {
351  o["owner_"] = o["owner"].as_string();
352  o.erase("owner");
353  }
354 
355  for( const auto& pair : original_arrays )
356  {
357  const auto& name = pair.first;
358  auto& value = pair.second;
359  auto type = data_type::map_type;
360  if( to_string_fields.find(name) != to_string_fields.end() )
361  type = to_string_fields.at(name);
362  o[name + "_object"] = adapt( value, type, max_depth - 1 );
363  }
364 
365  fc::variant v;
367  return v;
368 }
369 
370 fc::variant es_data_adaptor::adapt( const fc::variants& v, data_type type, uint16_t max_depth )
371 {
372  if( data_type::static_variant_type == type )
373  return adapt_static_variant( v, max_depth );
374 
375  // map_type or array_type
376  fc::variants vs;
377  vs.reserve( v.size() );
378  for( const auto& item : v )
379  {
380  if( item.is_array() )
381  {
382  if( data_type::map_type == type )
383  vs.push_back( adapt_map_item( item.get_array(), max_depth ) );
384  else // assume it is a static_variant array
385  vs.push_back( adapt_static_variant( item.get_array(), max_depth ) );
386  }
387  else if( item.is_object() ) // object array
388  vs.push_back( adapt( item.get_object(), max_depth ) );
389  else
390  wlog( "Type of item is unexpected: ${item}", ("item", item) );
391  }
392 
393  fc::variant nv;
395  return nv;
396 }
397 
399  const fc::variant& v, fc::mutable_variant_object& mv, const std::string& prefix, uint16_t max_depth )
400 {
401  FC_ASSERT( max_depth > 0, "Internal error" );
402  if( v.is_object() )
403  mv[prefix + "_object"] = adapt( v.get_object(), max_depth - 1 );
404  else if( v.is_int64() || v.is_uint64() )
405  mv[prefix + "_int"] = v;
406  else if( v.is_bool() )
407  mv[prefix + "_bool"] = v;
408  else if( v.is_string() )
409  mv[prefix + "_string"] = v.get_string();
410  else
411  mv[prefix + "_string"] = fc::json::to_string( v );
412  // Note: we don't use double here, and we convert nulls and blobs to strings,
413  // arrays and pairs (i.e. in custom authorities) are converted to strings,
414  // static_variants and maps (if any) are converted to strings too.
415 }
416 
418 {
419  if( 0 == max_depth )
420  {
421  fc::variant nv;
423  return nv;
424  }
425 
426  FC_ASSERT( v.size() == 2, "Internal error" );
428 
429  extract_data_from_variant( v[0], mv, "key", max_depth );
430  extract_data_from_variant( v[1], mv, "data", max_depth );
431 
432  fc::variant nv;
434  return nv;
435 }
436 
438 {
439  if( 0 == max_depth )
440  {
441  fc::variant nv;
443  return nv;
444  }
445 
446  FC_ASSERT( v.size() == 2, "Internal error" );
448 
449  mv["which"] = v[0];
450  extract_data_from_variant( v[1], mv, "data", max_depth );
451 
452  fc::variant nv;
454  return nv;
455 }
456 
457 void es_data_adaptor::in_situ_adapt( fc::variants& v, uint16_t max_depth )
458 {
459  for( auto& array_element : v )
460  {
461  if( array_element.is_object() )
462  array_element = adapt( array_element.get_object(), max_depth );
463  else if( array_element.is_array() )
464  in_situ_adapt( array_element.get_array(), max_depth );
465  else
466  array_element = array_element.as_string();
467  }
468 }
469 
470 } } // end namespace graphene::utilities
static string to_string(const variant &v, output_formatting format=stringify_large_ints_and_doubles, uint32_t max_depth=DEFAULT_MAX_RECURSION_DEPTH)
Definition: json.cpp:650
static fc::variant adapt_static_variant(const fc::variants &v, uint16_t max_depth)
http_response request(http_request_method method, const std::string &url, const std::string &auth, const std::string &query) const
http_response del(const std::string &url, const std::string &auth) const
An order-perserving dictionary of variant&#39;s.
Definition: api.cpp:48
#define elog(FORMAT,...)
Definition: logger.hpp:129
bool is_int64() const
Definition: variant.cpp:330
std::vector< variant > variants
Definition: variant.hpp:170
bool is_bool() const
Definition: variant.cpp:318
bool del(const std::string &path) const
bool is_string() const
Definition: variant.cpp:314
std::string query(const std::string &path, const std::string &query) const
std::vector< std::string > createBulk(const fc::mutable_variant_object &bulk_header, std::string &&data)
#define wlog(FORMAT,...)
Definition: logger.hpp:123
http_response post(const std::string &url, const std::string &auth, const std::string &query) const
void check_version_7_or_above(bool &result) const noexcept
variant_object & get_object()
Definition: variant.cpp:554
#define FC_THROW( ...)
Definition: exception.hpp:366
static fc::variant adapt_map_item(const fc::variants &v, uint16_t max_depth)
http_response put(const std::string &url, const std::string &auth, const std::string &query) const
void to_variant(const flat_set< T, A... > &var, variant &vo, uint32_t _max_depth)
Definition: flat.hpp:105
std::string get_version() const
#define FC_CAPTURE_AND_RETHROW(...)
Definition: exception.hpp:479
#define FC_ASSERT(TEST,...)
Checks a condition and throws an assert_exception if the test is FALSE.
Definition: exception.hpp:345
static fc::variant adapt(const fc::variant_object &op, uint16_t max_depth)
stores null, int64, uint64, double, bool, string, std::vector<variant>, and variant_object&#39;s.
Definition: variant.hpp:198
bool is_uint64() const
Definition: variant.cpp:326
http_response get(const std::string &url, const std::string &auth) const
Defines exception&#39;s used by fc.
static void extract_data_from_variant(const fc::variant &v, fc::mutable_variant_object &mv, const std::string &prefix, uint16_t max_depth)
Extract data from v into mv.
std::string to_string(double)
Definition: string.cpp:73
static variant from_string(const string &utf8_str, parse_type ptype=legacy_parser, uint32_t max_depth=DEFAULT_MAX_RECURSION_DEPTH)
Definition: json.cpp:458
const std::string & get_string() const
Definition: variant.cpp:575
#define FC_PACK_MAX_DEPTH
Definition: config.hpp:3
bool is_object() const
Definition: variant.cpp:363
bool send_bulk(const std::vector< std::string > &bulk_lines) const
bool as_bool() const
Definition: variant.cpp:441
static void in_situ_adapt(fc::variants &v, uint16_t max_depth)
Update directly, no return.
An order-perserving dictionary of variant&#39;s.
std::string as_string() const
Definition: variant.cpp:469
std::string get(const std::string &path) const