Update blockchain utilities with portable bootstrap file format

Remove repeated coinbase tx in each exported block's data.

Add resume from last exported height to blockchain_export, making it the
default behavior when the file already exists.

Start reorganizing the utilities.

Various cleanup.

Update output, including referring to both height and block numbers as
zero-based instead of one-based. This better matches the block data,
rather than just some parts of the existing codebase.

Use smaller default batch sizes for importer when verifying, so progress
is saved more frequently.

Use small default batch size (1000) for importer on Windows, due to
current issue with big transaction sizes on LMDB.

file format
-----------
[4-byte magic | variable-length header | block data]

header
------
4-byte file_info length
file_info struct
  file format major version
  file format minor version
  header length (includes file_info struct)
[rest of header, padded with 0 bytes up to header length]

block data
----------
4-byte chunk/block_package length
block_package struct
  block
  txs (coinbase/miner tx included already in block)
  block_size
  cumulative_difficulty
  coins_generated
4-byte chunk/block_package length
block_package struct
[...]

1 files changed, 4 insertions, 254 deletions

diff --git a/src/blockchain_converter/blockchain_export.cpp b/src/blockchain_converter/blockchain_export.cpp
index 8d7c78be3..aa34ea1dc 100644
--- a/src/blockchain_converter/blockchain_export.cpp
+++ b/src/blockchain_converter/blockchain_export.cpp
@@ -26,264 +26,14 @@
 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include <algorithm>
-#include <cstdio>
-#include <fstream>
-#include <boost/iostreams/copy.hpp>
-#include <atomic>
-
-#include <boost/archive/binary_oarchive.hpp>
-#include <boost/archive/binary_iarchive.hpp>
-#include <boost/iostreams/stream_buffer.hpp>
-#include <boost/iostreams/stream.hpp>
-#include <boost/iostreams/device/back_inserter.hpp>
-#include <boost/iostreams/filtering_streambuf.hpp>
-#include <boost/iostreams/filter/bzip2.hpp>
+#include "bootstrap_file.h"
 #include "common/command_line.h"
 #include "version.h"
-#include "blockchain_export.h"
-#include "cryptonote_core/cryptonote_boost_serialization.h"
-
-#include "import.h"
 
 unsigned int epee::g_test_dbg_lock_sleep = 0;
 
-static int max_chunk = 0;
-static size_t height;
-
 namespace po = boost::program_options;
-
-using namespace cryptonote;
-using namespace epee;
-
-bool BlockchainExport::open(const boost::filesystem::path& dir_path)
-{
-  if (boost::filesystem::exists(dir_path))
-  {
-    if (!boost::filesystem::is_directory(dir_path))
-    {
-      LOG_PRINT_RED_L0("export directory path is a file: " << dir_path);
-      return false;
-    }
-  }
-  else
-  {
-    if (!boost::filesystem::create_directory(dir_path))
-    {
-      LOG_PRINT_RED_L0("Failed to create directory " << dir_path);
-      return false;
-    }
-  }
-
-  std::string file_path = (dir_path / BLOCKCHAIN_RAW).string();
-  m_raw_data_file = new std::ofstream();
-  m_raw_data_file->open(file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc);
-  if (m_raw_data_file->fail())
-    return false;
-
-  m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
-  m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream);
-  if (m_raw_archive == NULL)
-    return false;
-
-  return true;
-}
-
-void BlockchainExport::flush_chunk()
-{
-  m_output_stream->flush();
-  char buffer[STR_LENGTH_OF_INT + 1];
-  int chunk_size = (int) m_buffer.size();
-  if (chunk_size > BUFFER_SIZE)
-  {
-    LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
-  }
-  sprintf(buffer, STR_FORMAT_OF_INT, chunk_size);
-  m_raw_data_file->write(buffer, STR_LENGTH_OF_INT);
-  if (max_chunk < chunk_size)
-  {
-    max_chunk = chunk_size;
-  }
-  long pos_before = m_raw_data_file->tellp();
-  std::copy(m_buffer.begin(), m_buffer.end(), std::ostreambuf_iterator<char>(*m_raw_data_file));
-  m_raw_data_file->flush();
-  long pos_after = m_raw_data_file->tellp();
-  long num_chars_written = pos_after - pos_before;
-  if ((int) num_chars_written != chunk_size)
-  {
-    LOG_PRINT_RED_L0("INTERNAL ERROR: num chars wrote NEQ buffer size. height = " << height);
-  }
-
-  m_buffer.clear();
-  delete m_raw_archive;
-  delete m_output_stream;
-  m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
-  m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream);
-}
-
-void BlockchainExport::serialize_block_to_text_buffer(const block& block)
-{
-  *m_raw_archive << block;
-}
-
-void BlockchainExport::buffer_serialize_tx(const transaction& tx)
-{
-  *m_raw_archive << tx;
-}
-
-void BlockchainExport::buffer_write_num_txs(const std::list<transaction> txs)
-{
-  int n = txs.size();
-  *m_raw_archive << n;
-}
-
-void BlockchainExport::write_block(block& block)
-{
-  serialize_block_to_text_buffer(block);
-  std::list<transaction> txs;
-
-  uint64_t block_height = boost::get<txin_gen>(block.miner_tx.vin.front()).height;
-
-  // put coinbase transaction first
-  transaction coinbase_tx = block.miner_tx;
-  crypto::hash coinbase_tx_hash = get_transaction_hash(coinbase_tx);
-#if SOURCE_DB == DB_MEMORY
-  const transaction* cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash);
-#else
-  transaction cb_tx_full = m_blockchain_storage->get_db().get_tx(coinbase_tx_hash);
-#endif
-
-#if SOURCE_DB == DB_MEMORY
-  if (cb_tx_full != NULL)
-  {
-    txs.push_back(*cb_tx_full);
-  }
-#else
-  // TODO: should check and abort if cb_tx_full equals null_hash?
-  txs.push_back(cb_tx_full);
-#endif
-
-  // now add all regular transactions
-  BOOST_FOREACH(const auto& tx_id, block.tx_hashes)
-  {
-#if SOURCE_DB == DB_MEMORY
-    const transaction* tx = m_blockchain_storage->get_tx(tx_id);
-#else
-    transaction tx = m_blockchain_storage->get_db().get_tx(tx_id);
-#endif
-
-#if SOURCE_DB == DB_MEMORY
-    if(tx == NULL)
-    {
-      if (! m_tx_pool)
-        throw std::runtime_error("Aborting: tx == NULL, so memory pool required to get tx, but memory pool isn't enabled");
-      else
-      {
-        transaction tx;
-        if(m_tx_pool->get_transaction(tx_id, tx))
-          txs.push_back(tx);
-        else
-          throw std::runtime_error("Aborting: tx not found in pool");
-      }
-    }
-    else
-      txs.push_back(*tx);
-#else
-    txs.push_back(tx);
-#endif
-  }
-
-  // serialize all txs to the persistant storage
-  buffer_write_num_txs(txs);
-  BOOST_FOREACH(const auto& tx, txs)
-  {
-    buffer_serialize_tx(tx);
-  }
-
-  // These three attributes are currently necessary for a fast import that adds blocks without verification.
-  bool include_extra_block_data = true;
-  if (include_extra_block_data)
-  {
-#if SOURCE_DB == DB_MEMORY
-    size_t block_size = m_blockchain_storage->get_block_size(block_height);
-    difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height);
-    uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height);
-#else
-    size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height);
-    difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height);
-    uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height);
-#endif
-
-    *m_raw_archive << block_size;
-    *m_raw_archive << cumulative_difficulty;
-    *m_raw_archive << coins_generated;
-  }
-}
-
-bool BlockchainExport::BlockchainExport::close()
-{
-  if (m_raw_data_file->fail())
-    return false;
-
-  m_raw_data_file->flush();
-  delete m_raw_archive;
-  delete m_output_stream;
-  delete m_raw_data_file;
-  return true;
-}
-
-
-#if SOURCE_DB == DB_MEMORY
-bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height)
-#else
-bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height)
-#endif
-{
-  uint64_t block_height = 0;
-  m_blockchain_storage = _blockchain_storage;
-  m_tx_pool = _tx_pool;
-  uint64_t progress_interval = 100;
-  std::string refresh_string = "\r                                    \r";
-  LOG_PRINT_L0("Storing blocks raw data...");
-  if (!BlockchainExport::open(output_dir))
-  {
-    LOG_PRINT_RED_L0("failed to open raw file for write");
-    return false;
-  }
-  block b;
-  LOG_PRINT_L0("source blockchain height: " <<  m_blockchain_storage->get_current_blockchain_height());
-  LOG_PRINT_L0("requested block height: " << requested_block_height);
-  if ((requested_block_height > 0) && (requested_block_height < m_blockchain_storage->get_current_blockchain_height()))
-    block_height = requested_block_height;
-  else
-  {
-    block_height = m_blockchain_storage->get_current_blockchain_height();
-    LOG_PRINT_L0("Using block height of source blockchain: " << block_height);
-  }
-  for (height=0; height < block_height; ++height)
-  {
-    crypto::hash hash = m_blockchain_storage->get_block_id_by_height(height);
-    m_blockchain_storage->get_block_by_hash(hash, b);
-    write_block(b);
-    if (height % NUM_BLOCKS_PER_CHUNK == 0) {
-      flush_chunk();
-    }
-    if (height % progress_interval == 0) {
-      std::cout << refresh_string;
-      std::cout << "height " << height << "/" << block_height << std::flush;
-    }
-  }
-  if (height % NUM_BLOCKS_PER_CHUNK != 0)
-  {
-    flush_chunk();
-  }
-  std::cout << refresh_string;
-  std::cout << "height " << height << "/" << block_height << ENDL;
-
-  LOG_PRINT_L0("longest chunk was " << max_chunk << " bytes");
-  return BlockchainExport::close();
-}
-
+using namespace epee; // log_space
 
 int main(int argc, char* argv[])
 {
@@ -397,8 +147,8 @@ int main(int argc, char* argv[])
   LOG_PRINT_L0("Source blockchain storage initialized OK");
   LOG_PRINT_L0("Exporting blockchain raw data...");
 
-  BlockchainExport be;
-  r = be.store_blockchain_raw(core_storage, NULL, output_dir, block_height);
+  BootstrapFile bootstrap;
+  r = bootstrap.store_blockchain_raw(core_storage, NULL, output_dir, block_height);
   CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data");
   LOG_PRINT_L0("Blockchain raw data exported OK");
 }