BitShares-Core  4.0.0
BitShares blockchain implementation and command-line interface software
elasticsearch.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 oxarbitrage, and contributors.
3  *
4  * The MIT License
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
25 
26 #include <boost/algorithm/string/join.hpp>
27 #include <boost/algorithm/string.hpp>
28 #include <fc/log/logger.hpp>
29 #include <fc/io/json.hpp>
30 
31 size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
32 {
33  ((std::string*)userp)->append((char*)contents, size * nmemb);
34  return size * nmemb;
35 }
36 
37 namespace graphene { namespace utilities {
38 
39 bool checkES(ES& es)
40 {
42  curl_request.handler = es.curl;
43  curl_request.url = es.elasticsearch_url + "_nodes";
44  curl_request.auth = es.auth;
45  curl_request.type = "GET";
46 
47  if(doCurl(curl_request).empty())
48  return false;
49  return true;
50 
51 }
52 const std::string simpleQuery(ES& es)
53 {
55  curl_request.handler = es.curl;
56  curl_request.url = es.elasticsearch_url + es.endpoint;
57  curl_request.auth = es.auth;
58  curl_request.type = "POST";
59  curl_request.query = es.query;
60 
61  return doCurl(curl_request);
62 }
63 
64 bool SendBulk(ES&& es)
65 {
66  std::string bulking = joinBulkLines(es.bulk_lines);
67 
69  curl_request.handler = es.curl;
70  curl_request.url = es.elasticsearch_url + "_bulk";
71  curl_request.auth = es.auth;
72  curl_request.type = "POST";
73  curl_request.query = std::move(bulking);
74 
75  auto curlResponse = doCurl(curl_request);
76 
77  if(handleBulkResponse(getResponseCode(curl_request.handler), curlResponse))
78  return true;
79  return false;
80 }
81 
82 const std::string joinBulkLines(const std::vector<std::string>& bulk)
83 {
84  auto bulking = boost::algorithm::join(bulk, "\n");
85  bulking = bulking + "\n";
86 
87  return bulking;
88 }
89 long getResponseCode(CURL *handler)
90 {
91  long http_code = 0;
92  curl_easy_getinfo (handler, CURLINFO_RESPONSE_CODE, &http_code);
93  return http_code;
94 }
95 
96 bool handleBulkResponse(long http_code, const std::string& CurlReadBuffer)
97 {
98  if(http_code == 200) {
99  // all good, but check errors in response
100  fc::variant j = fc::json::from_string(CurlReadBuffer);
101  bool errors = j["errors"].as_bool();
102  if(errors == true) {
103  return false;
104  }
105  }
106  else {
107  if(http_code == 413) {
108  elog( "413 error: Can be low disk space" );
109  }
110  else if(http_code == 401) {
111  elog( "401 error: Unauthorized" );
112  }
113  else {
114  elog( std::to_string(http_code) + " error: Unknown error" );
115  }
116  return false;
117  }
118  return true;
119 }
120 
121 const std::vector<std::string> createBulk(const fc::mutable_variant_object& bulk_header, std::string&& data)
122 {
123  std::vector<std::string> bulk;
124  fc::mutable_variant_object final_bulk_header;
125  final_bulk_header["index"] = bulk_header;
126  bulk.push_back(fc::json::to_string(final_bulk_header));
127  bulk.push_back(data);
128 
129  return bulk;
130 }
131 
133 {
135  curl_request.handler = es.curl;
136  curl_request.url = es.elasticsearch_url + es.index_prefix + "*";
137  curl_request.auth = es.auth;
138  curl_request.type = "DELETE";
139 
140  auto curl_response = doCurl(curl_request);
141  if(curl_response.empty())
142  return false;
143  else
144  return true;
145 }
146 const std::string getEndPoint(ES& es)
147 {
149  curl_request.handler = es.curl;
150  curl_request.url = es.elasticsearch_url + es.endpoint;
151  curl_request.auth = es.auth;
152  curl_request.type = "GET";
153 
154  return doCurl(curl_request);
155 }
156 
157 const std::string generateIndexName(const fc::time_point_sec& block_date, const std::string& _elasticsearch_index_prefix)
158 {
159  auto block_date_string = block_date.to_iso_string();
160  std::vector<std::string> parts;
161  boost::split(parts, block_date_string, boost::is_any_of("-"));
162  std::string index_name = _elasticsearch_index_prefix + parts[0] + "-" + parts[1];
163  return index_name;
164 }
165 
166 const std::string doCurl(CurlRequest& curl)
167 {
168  std::string CurlReadBuffer;
169  struct curl_slist *headers = NULL;
170  headers = curl_slist_append(headers, "Content-Type: application/json");
171 
172  curl_easy_setopt(curl.handler, CURLOPT_HTTPHEADER, headers);
173  curl_easy_setopt(curl.handler, CURLOPT_URL, curl.url.c_str());
174  curl_easy_setopt(curl.handler, CURLOPT_CUSTOMREQUEST, curl.type.c_str());
175  if(curl.type == "POST")
176  {
177  curl_easy_setopt(curl.handler, CURLOPT_POST, true);
178  curl_easy_setopt(curl.handler, CURLOPT_POSTFIELDS, curl.query.c_str());
179  }
180  curl_easy_setopt(curl.handler, CURLOPT_WRITEFUNCTION, WriteCallback);
181  curl_easy_setopt(curl.handler, CURLOPT_WRITEDATA, (void *)&CurlReadBuffer);
182  curl_easy_setopt(curl.handler, CURLOPT_USERAGENT, "libcrp/0.1");
183  if(!curl.auth.empty())
184  curl_easy_setopt(curl.handler, CURLOPT_USERPWD, curl.auth.c_str());
185  curl_easy_perform(curl.handler);
186 
187  return CurlReadBuffer;
188 }
189 
190 } } // end namespace graphene::utilities
static string to_string(const variant &v, output_formatting format=stringify_large_ints_and_doubles, uint32_t max_depth=DEFAULT_MAX_RECURSION_DEPTH)
Definition: json.cpp:638
long getResponseCode(CURL *handler)
bool handleBulkResponse(long http_code, const std::string &CurlReadBuffer)
Definition: api.cpp:56
#define elog(FORMAT,...)
Definition: logger.hpp:129
bool as_bool() const
Definition: variant.cpp:441
bool SendBulk(ES &&es)
const std::string doCurl(CurlRequest &curl)
const std::vector< std::string > createBulk(const fc::mutable_variant_object &bulk_header, std::string &&data)
const std::string simpleQuery(ES &es)
size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
bool deleteAll(ES &es)
stores null, int64, uint64, double, bool, string, std::vector<variant>, and variant_object&#39;s.
Definition: variant.hpp:198
bool checkES(ES &es)
const std::string getEndPoint(ES &es)
const std::string joinBulkLines(const std::vector< std::string > &bulk)
std::string to_string(double)
Definition: string.cpp:73
std::string to_iso_string() const
Definition: time.cpp:24
std::vector< header > headers
Definition: connection.hpp:21
static variant from_string(const string &utf8_str, parse_type ptype=legacy_parser, uint32_t max_depth=DEFAULT_MAX_RECURSION_DEPTH)
Definition: json.cpp:458
const std::string generateIndexName(const fc::time_point_sec &block_date, const std::string &_elasticsearch_index_prefix)
An order-perserving dictionary of variant&#39;s.