BitShares-Core  4.0.0
BitShares blockchain implementation and command-line interface software
db_management.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cryptonomex, Inc., and contributors.
3  *
4  * The MIT License
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 
26 
31 
33 
34 #include <fc/io/fstream.hpp>
35 
36 #include <fstream>
37 #include <functional>
38 #include <iostream>
39 #include <queue>
40 #include <tuple>
41 
42 namespace graphene { namespace chain {
43 
45 {
48 }
49 
51 {
52  clear_pending();
53 }
54 
55 void database::reindex( fc::path data_dir )
56 { try {
57  auto last_block = _block_id_to_block.last();
58  if( !last_block ) {
59  elog( "!no last block" );
60  edump((last_block));
61  return;
62  }
63  if( last_block->block_num() <= head_block_num()) return;
64 
65  ilog( "reindexing blockchain" );
66  auto start = fc::time_point::now();
67  const auto last_block_num = last_block->block_num();
68  uint32_t undo_point = last_block_num < GRAPHENE_MAX_UNDO_HISTORY ? 0 : last_block_num - GRAPHENE_MAX_UNDO_HISTORY;
69 
70  ilog( "Replaying blocks, starting at ${next}...", ("next",head_block_num() + 1) );
71  if( head_block_num() >= undo_point )
72  {
73  if( head_block_num() > 0 )
75  }
76  else
77  _undo_db.disable();
78 
79  uint32_t skip = node_properties().skip_flags;
80 
81  size_t total_block_size = _block_id_to_block.total_block_size();
82  const auto& gpo = get_global_properties();
83  std::queue< std::tuple< size_t, signed_block, fc::future< void > > > blocks;
84  uint32_t next_block_num = head_block_num() + 1;
85  uint32_t i = next_block_num;
86  while( next_block_num <= last_block_num || !blocks.empty() )
87  {
88  if( next_block_num <= last_block_num && blocks.size() < 20 )
89  {
90  const size_t processed_block_size = _block_id_to_block.blocks_current_position();
91  fc::optional< signed_block > block = _block_id_to_block.fetch_by_number( next_block_num++ );
92  if( block.valid() )
93  {
94  if( block->timestamp >= last_block->timestamp - gpo.parameters.maximum_time_until_expiration )
96  blocks.emplace( processed_block_size, std::move(*block), fc::future<void>() );
97  std::get<2>(blocks.back()) = precompute_parallel( std::get<1>(blocks.back()), skip );
98  }
99  else
100  {
101  wlog( "Reindexing terminated due to gap: Block ${i} does not exist!", ("i", i) );
102  uint32_t dropped_count = 0;
103  while( true )
104  {
105  fc::optional< block_id_type > last_id = _block_id_to_block.last_id();
106  // this can trigger if we attempt to e.g. read a file that has block #2 but no block #1
107  if( !last_id.valid() )
108  break;
109  // we've caught up to the gap
110  if( block_header::num_from_id( *last_id ) <= i )
111  break;
112  _block_id_to_block.remove( *last_id );
113  dropped_count++;
114  }
115  wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) );
116  next_block_num = last_block_num + 1; // don't load more blocks
117  }
118  }
119  else
120  {
121  std::get<2>(blocks.front()).wait();
122  const signed_block& block = std::get<1>(blocks.front());
123 
124  if( i % 10000 == 0 )
125  {
126  std::stringstream bysize;
127  std::stringstream bynum;
128  bysize << std::fixed << std::setprecision(5) << double(std::get<0>(blocks.front())) / total_block_size * 100;
129  bynum << std::fixed << std::setprecision(5) << double(i*100)/last_block_num;
130  ilog(
131  " [by size: ${size}% ${processed} of ${total}] [by num: ${num}% ${i} of ${last}]",
132  ("size", bysize.str())
133  ("processed", std::get<0>(blocks.front()))
134  ("total", total_block_size)
135  ("num", bynum.str())
136  ("i", i)
137  ("last", last_block_num)
138  );
139  }
140  if( i == undo_point )
141  {
142  ilog( "Writing database to disk at block ${i}", ("i",i) );
143  flush();
144  ilog( "Done" );
145  }
146  if( i < undo_point )
147  apply_block( block, skip );
148  else
149  {
150  _undo_db.enable();
151  push_block( block, skip );
152  }
153  blocks.pop();
154  i++;
155  }
156  }
157  _undo_db.enable();
158  auto end = fc::time_point::now();
159  ilog( "Done reindexing, elapsed time: ${t} sec", ("t",double((end-start).count())/1000000.0 ) );
160 } FC_CAPTURE_AND_RETHROW( (data_dir) ) }
161 
162 void database::wipe(const fc::path& data_dir, bool include_blocks)
163 {
164  ilog("Wiping database", ("include_blocks", include_blocks));
165  if (_opened) {
166  close();
167  }
168  object_database::wipe(data_dir);
169  if( include_blocks )
170  fc::remove_all( data_dir / "database" );
171 }
172 
174  const fc::path& data_dir,
175  std::function<genesis_state_type()> genesis_loader,
176  const std::string& db_version)
177 {
178  try
179  {
180  bool wipe_object_db = false;
181  if( !fc::exists( data_dir / "db_version" ) )
182  wipe_object_db = true;
183  else
184  {
185  std::string version_string;
186  fc::read_file_contents( data_dir / "db_version", version_string );
187  wipe_object_db = ( version_string != db_version );
188  }
189  if( wipe_object_db ) {
190  ilog("Wiping object_database due to missing or wrong version");
191  object_database::wipe( data_dir );
192  std::ofstream version_file( (data_dir / "db_version").generic_string().c_str(),
193  std::ios::out | std::ios::binary | std::ios::trunc );
194  version_file.write( db_version.c_str(), db_version.size() );
195  version_file.close();
196  }
197 
198  object_database::open(data_dir);
199 
200  _block_id_to_block.open(data_dir / "database" / "block_num_to_block");
201 
202  if( !find(global_property_id_type()) )
203  init_genesis(genesis_loader());
204  else
205  {
206  _p_core_asset_obj = &get( asset_id_type() );
207  _p_core_dynamic_data_obj = &get( asset_dynamic_data_id_type() );
208  _p_global_prop_obj = &get( global_property_id_type() );
209  _p_chain_property_obj = &get( chain_property_id_type() );
210  _p_dyn_global_prop_obj = &get( dynamic_global_property_id_type() );
211  _p_witness_schedule_obj = &get( witness_schedule_id_type() );
212  }
213 
214  fc::optional<block_id_type> last_block = _block_id_to_block.last_id();
215  if( last_block.valid() )
216  {
217  FC_ASSERT( *last_block >= head_block_id(),
218  "last block ID does not match current chain state",
219  ("last_block->id", last_block)("head_block_id",head_block_num()) );
220  reindex( data_dir );
221  }
222  _opened = true;
223  }
224  FC_CAPTURE_LOG_AND_RETHROW( (data_dir) )
225 }
226 
227 void database::close(bool rewind)
228 {
229  if (!_opened)
230  return;
231 
232  // TODO: Save pending tx's on close()
233  clear_pending();
234 
235  // pop all of the blocks that we can given our undo history, this should
236  // throw when there is no more undo history to pop
237  if( rewind )
238  {
239  try
240  {
242 
243  ilog( "Rewinding from ${head} to ${cutoff}", ("head",head_block_num())("cutoff",cutoff) );
244  while( head_block_num() > cutoff )
245  {
246  block_id_type popped_block_id = head_block_id();
247  pop_block();
248  _fork_db.remove(popped_block_id); // doesn't throw on missing
249  }
250  }
251  catch ( const fc::exception& e )
252  {
253  wlog( "Database close unexpected exception: ${e}", ("e", e) );
254  }
255  }
256 
257  // Since pop_block() will move tx's in the popped blocks into pending,
258  // we have to clear_pending() after we're done popping to get a clean
259  // DB state (issue #336).
260  clear_pending();
261 
264 
265  if( _block_id_to_block.is_open() )
266  _block_id_to_block.close();
267 
268  _fork_db.reset();
269 
270  _opened = false;
271 }
272 
273 } }
bool exists(const path &p)
Definition: filesystem.cpp:209
void reindex(fc::path data_dir)
Rebuild object graph from block history and open detabase.
void wipe(const fc::path &data_dir, bool include_blocks)
wipe Delete database from disk, and potentially the raw chain as well.
const T * find(object_id_type id) const
static uint32_t num_from_id(const block_id_type &id)
Definition: block.cpp:36
T wait(boost::signals2::signal< void(T)> &sig, const microseconds &timeout_us=microseconds::maximum())
Definition: signals.hpp:38
void init_genesis(const genesis_state_type &genesis_state=genesis_state_type())
Definition: db_init.cpp:242
Definition: api.cpp:56
#define elog(FORMAT,...)
Definition: logger.hpp:129
void start_block(signed_block b)
fc::future< void > precompute_parallel(const signed_block &block, const uint32_t skip=skip_nothing) const
Definition: db_block.cpp:829
Used to generate a useful error report when an exception is thrown.At each level in the stack where t...
Definition: exception.hpp:56
const dynamic_global_property_object & get_dynamic_global_properties() const
Definition: db_getter.cpp:54
bool valid() const
Definition: optional.hpp:186
uint32_t head_block_num() const
Definition: db_getter.cpp:69
void open(const fc::path &dbdir)
void remove_all(const path &p)
Definition: filesystem.cpp:240
void read_file_contents(const fc::path &filename, std::string &result)
Definition: fstream.cpp:107
#define wlog(FORMAT,...)
Definition: logger.hpp:123
bool push_block(const signed_block &b, uint32_t skip=skip_nothing)
Definition: db_block.cpp:116
const global_property_object & get_global_properties() const
Definition: db_getter.cpp:44
optional< signed_block > fetch_block_by_number(uint32_t num) const
Definition: db_block.cpp:75
provides stack-based nullable value similar to boost::optional
Definition: optional.hpp:20
optional< block_id_type > last_id() const
void apply_block(const signed_block &next_block, uint32_t skip=skip_nothing)
Definition: db_block.cpp:555
void open(const fc::path &data_dir, std::function< genesis_state_type()> genesis_loader, const std::string &db_version)
Open a database, creating a new one if necessary.
#define edump(SEQ)
Definition: logger.hpp:182
void remove(const block_id_type &id)
#define ilog(FORMAT,...)
Definition: logger.hpp:117
#define FC_CAPTURE_AND_RETHROW(...)
Definition: exception.hpp:478
#define FC_ASSERT(TEST,...)
Checks a condition and throws an assert_exception if the test is FALSE.
Definition: exception.hpp:345
void wipe(const fc::path &data_dir)
void initialize_indexes()
Reset the object graph in-memory.
Definition: db_init.cpp:197
void remove(block_id_type b)
void open(const fc::path &data_dir)
node_property_object & node_properties()
Definition: db_getter.cpp:94
optional< signed_block > fetch_by_number(uint32_t block_num) const
static time_point now()
Definition: time.cpp:13
#define FC_CAPTURE_LOG_AND_RETHROW(...)
Definition: exception.hpp:414
block_id_type head_block_id() const
Definition: db_getter.cpp:74
wraps boost::filesystem::path to provide platform independent path manipulation.
Definition: filesystem.hpp:28
optional< signed_block > last() const
#define GRAPHENE_MAX_UNDO_HISTORY
Definition: config.hpp:29