BitShares-Core  5.0.0
BitShares blockchain implementation and command-line interface software
db_management.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cryptonomex, Inc., and contributors.
3  *
4  * The MIT License
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 
26 
31 
33 
34 #include <fc/io/fstream.hpp>
35 
36 #include <fstream>
37 #include <functional>
38 #include <iostream>
39 #include <queue>
40 #include <tuple>
41 
42 namespace graphene { namespace chain {
43 
45 {
48 }
49 
51 {
52  clear_pending();
53 }
54 
55 void database::reindex( fc::path data_dir )
56 { try {
57  auto last_block = _block_id_to_block.last();
58  if( !last_block ) {
59  elog( "!no last block" );
60  edump((last_block));
61  return;
62  }
63  if( last_block->block_num() <= head_block_num()) return;
64 
65  ilog( "reindexing blockchain" );
66  auto start = fc::time_point::now();
67  const auto last_block_num = last_block->block_num();
68  uint32_t undo_point = last_block_num < GRAPHENE_MAX_UNDO_HISTORY ? 0 : last_block_num - GRAPHENE_MAX_UNDO_HISTORY;
69 
70  ilog( "Replaying blocks, starting at ${next}...", ("next",head_block_num() + 1) );
71  if( head_block_num() >= undo_point )
72  {
73  if( head_block_num() > 0 )
75  }
76  else
77  _undo_db.disable();
78 
79  uint32_t skip = node_properties().skip_flags;
80 
81  size_t total_block_size = _block_id_to_block.total_block_size();
82  const auto& gpo = get_global_properties();
83  std::queue< std::tuple< size_t, signed_block, fc::future< void > > > blocks;
84  uint32_t next_block_num = head_block_num() + 1;
85  uint32_t i = next_block_num;
86  while( next_block_num <= last_block_num || !blocks.empty() )
87  {
88  if( next_block_num <= last_block_num && blocks.size() < 20 )
89  {
90  const size_t processed_block_size = _block_id_to_block.blocks_current_position();
91  fc::optional< signed_block > block = _block_id_to_block.fetch_by_number( next_block_num++ );
92  if( block.valid() )
93  {
94  if( block->timestamp >= last_block->timestamp - gpo.parameters.maximum_time_until_expiration )
96  blocks.emplace( processed_block_size, std::move(*block), fc::future<void>() );
97  std::get<2>(blocks.back()) = precompute_parallel( std::get<1>(blocks.back()), skip );
98  }
99  else
100  {
101  wlog( "Reindexing terminated due to gap: Block ${i} does not exist!", ("i", i) );
102  uint32_t dropped_count = 0;
103  while( true )
104  {
105  fc::optional< block_id_type > last_id = _block_id_to_block.last_id();
106  // this can trigger if we attempt to e.g. read a file that has block #2 but no block #1
107  if( !last_id.valid() )
108  break;
109  // we've caught up to the gap
110  if( block_header::num_from_id( *last_id ) <= i )
111  break;
112  _block_id_to_block.remove( *last_id );
113  dropped_count++;
114  }
115  wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) );
116  next_block_num = last_block_num + 1; // don't load more blocks
117  }
118  }
119  else
120  {
121  std::get<2>(blocks.front()).wait();
122  const signed_block& block = std::get<1>(blocks.front());
123 
124  if( i % 10000 == 0 )
125  {
126  std::stringstream bysize;
127  std::stringstream bynum;
128  size_t current_pos = std::get<0>(blocks.front());
129  if( current_pos > total_block_size )
130  total_block_size = current_pos;
131  bysize << std::fixed << std::setprecision(5) << double(current_pos) / total_block_size * 100;
132  bynum << std::fixed << std::setprecision(5) << double(i)*100/last_block_num;
133  ilog(
134  " [by size: ${size}% ${processed} of ${total}] [by num: ${num}% ${i} of ${last}]",
135  ("size", bysize.str())
136  ("processed", current_pos)
137  ("total", total_block_size)
138  ("num", bynum.str())
139  ("i", i)
140  ("last", last_block_num)
141  );
142  }
143  if( i == undo_point )
144  {
145  ilog( "Writing database to disk at block ${i}", ("i",i) );
146  flush();
147  ilog( "Done" );
148  }
149  if( i < undo_point )
150  apply_block( block, skip );
151  else
152  {
153  _undo_db.enable();
154  push_block( block, skip );
155  }
156  blocks.pop();
157  i++;
158  }
159  }
160  _undo_db.enable();
161  auto end = fc::time_point::now();
162  ilog( "Done reindexing, elapsed time: ${t} sec", ("t",double((end-start).count())/1000000.0 ) );
163 } FC_CAPTURE_AND_RETHROW( (data_dir) ) }
164 
165 void database::wipe(const fc::path& data_dir, bool include_blocks)
166 {
167  ilog("Wiping database", ("include_blocks", include_blocks));
168  if (_opened) {
169  close();
170  }
171  object_database::wipe(data_dir);
172  if( include_blocks )
173  fc::remove_all( data_dir / "database" );
174 }
175 
177  const fc::path& data_dir,
178  std::function<genesis_state_type()> genesis_loader,
179  const std::string& db_version)
180 {
181  try
182  {
183  bool wipe_object_db = false;
184  if( !fc::exists( data_dir / "db_version" ) )
185  wipe_object_db = true;
186  else
187  {
188  std::string version_string;
189  fc::read_file_contents( data_dir / "db_version", version_string );
190  wipe_object_db = ( version_string != db_version );
191  }
192  if( wipe_object_db ) {
193  ilog("Wiping object_database due to missing or wrong version");
194  object_database::wipe( data_dir );
195  std::ofstream version_file( (data_dir / "db_version").generic_string().c_str(),
196  std::ios::out | std::ios::binary | std::ios::trunc );
197  version_file.write( db_version.c_str(), db_version.size() );
198  version_file.close();
199  }
200 
201  object_database::open(data_dir);
202 
203  _block_id_to_block.open(data_dir / "database" / "block_num_to_block");
204 
205  if( !find(global_property_id_type()) )
206  init_genesis(genesis_loader());
207  else
208  {
209  _p_core_asset_obj = &get( asset_id_type() );
210  _p_core_dynamic_data_obj = &get( asset_dynamic_data_id_type() );
211  _p_global_prop_obj = &get( global_property_id_type() );
212  _p_chain_property_obj = &get( chain_property_id_type() );
213  _p_dyn_global_prop_obj = &get( dynamic_global_property_id_type() );
214  _p_witness_schedule_obj = &get( witness_schedule_id_type() );
215  }
216 
217  fc::optional<block_id_type> last_block = _block_id_to_block.last_id();
218  if( last_block.valid() )
219  {
220  FC_ASSERT( *last_block >= head_block_id(),
221  "last block ID does not match current chain state",
222  ("last_block->id", last_block)("head_block_id",head_block_num()) );
223  reindex( data_dir );
224  }
225  _opened = true;
226  }
227  FC_CAPTURE_LOG_AND_RETHROW( (data_dir) )
228 }
229 
230 void database::close(bool rewind)
231 {
232  if (!_opened)
233  return;
234 
235  // TODO: Save pending tx's on close()
236  clear_pending();
237 
238  // pop all of the blocks that we can given our undo history, this should
239  // throw when there is no more undo history to pop
240  if( rewind )
241  {
242  try
243  {
245 
246  ilog( "Rewinding from ${head} to ${cutoff}", ("head",head_block_num())("cutoff",cutoff) );
247  while( head_block_num() > cutoff )
248  {
249  block_id_type popped_block_id = head_block_id();
250  pop_block();
251  _fork_db.remove(popped_block_id); // doesn't throw on missing
252  }
253  }
254  catch ( const fc::exception& e )
255  {
256  wlog( "Database close unexpected exception: ${e}", ("e", e) );
257  }
258  }
259 
260  // Since pop_block() will move tx's in the popped blocks into pending,
261  // we have to clear_pending() after we're done popping to get a clean
262  // DB state (issue #336).
263  clear_pending();
264 
267 
268  if( _block_id_to_block.is_open() )
269  _block_id_to_block.close();
270 
271  _fork_db.reset();
272 
273  _opened = false;
274 }
275 
276 } }
bool exists(const path &p)
Definition: filesystem.cpp:209
void reindex(fc::path data_dir)
Rebuild object graph from block history and open detabase.
void wipe(const fc::path &data_dir, bool include_blocks)
wipe Delete database from disk, and potentially the raw chain as well.
const T * find(object_id_type id) const
static uint32_t num_from_id(const block_id_type &id)
Definition: block.cpp:36
T wait(boost::signals2::signal< void(T)> &sig, const microseconds &timeout_us=microseconds::maximum())
Definition: signals.hpp:38
void init_genesis(const genesis_state_type &genesis_state=genesis_state_type())
Definition: db_init.cpp:187
Definition: api.cpp:56
#define elog(FORMAT,...)
Definition: logger.hpp:129
void start_block(signed_block b)
fc::future< void > precompute_parallel(const signed_block &block, const uint32_t skip=skip_nothing) const
Definition: db_block.cpp:829
Used to generate a useful error report when an exception is thrown.At each level in the stack where t...
Definition: exception.hpp:56
const dynamic_global_property_object & get_dynamic_global_properties() const
Definition: db_getter.cpp:54
bool valid() const
Definition: optional.hpp:186
uint32_t head_block_num() const
Definition: db_getter.cpp:69
void open(const fc::path &dbdir)
void remove_all(const path &p)
Definition: filesystem.cpp:240
void read_file_contents(const fc::path &filename, std::string &result)
Definition: fstream.cpp:107
#define wlog(FORMAT,...)
Definition: logger.hpp:123
bool push_block(const signed_block &b, uint32_t skip=skip_nothing)
Definition: db_block.cpp:116
const global_property_object & get_global_properties() const
Definition: db_getter.cpp:44
optional< signed_block > fetch_block_by_number(uint32_t num) const
Definition: db_block.cpp:75
provides stack-based nullable value similar to boost::optional
Definition: optional.hpp:20
optional< block_id_type > last_id() const
void apply_block(const signed_block &next_block, uint32_t skip=skip_nothing)
Definition: db_block.cpp:555
void open(const fc::path &data_dir, std::function< genesis_state_type()> genesis_loader, const std::string &db_version)
Open a database, creating a new one if necessary.
#define edump(SEQ)
Definition: logger.hpp:182
void remove(const block_id_type &id)
#define ilog(FORMAT,...)
Definition: logger.hpp:117
#define FC_CAPTURE_AND_RETHROW(...)
Definition: exception.hpp:478
#define FC_ASSERT(TEST,...)
Checks a condition and throws an assert_exception if the test is FALSE.
Definition: exception.hpp:345
void wipe(const fc::path &data_dir)
void initialize_indexes()
Reset the object graph in-memory.
Definition: db_init.cpp:141
void remove(block_id_type b)
void open(const fc::path &data_dir)
node_property_object & node_properties()
Definition: db_getter.cpp:94
optional< signed_block > fetch_by_number(uint32_t block_num) const
static time_point now()
Definition: time.cpp:13
#define FC_CAPTURE_LOG_AND_RETHROW(...)
Definition: exception.hpp:414
block_id_type head_block_id() const
Definition: db_getter.cpp:74
wraps boost::filesystem::path to provide platform independent path manipulation.
Definition: filesystem.hpp:28
optional< signed_block > last() const
#define GRAPHENE_MAX_UNDO_HISTORY
Definition: config.hpp:29