Update blockchain utilities with portable bootstrap file format

Remove repeated coinbase tx in each exported block's data.

Add resume from last exported height to blockchain_export, making it the
default behavior when the file already exists.

Start reorganizing the utilities.

Various cleanup.

Update output, including referring to both height and block numbers as
zero-based instead of one-based. This better matches the block data,
rather than just some parts of the existing codebase.

Use smaller default batch sizes for importer when verifying, so progress
is saved more frequently.

Use small default batch size (1000) for importer on Windows, due to
current issue with big transaction sizes on LMDB.

file format
-----------
[4-byte magic | variable-length header | block data]

header
------
4-byte file_info length
file_info struct
  file format major version
  file format minor version
  header length (includes file_info struct)
[rest of header, padded with 0 bytes up to header length]

block data
----------
4-byte chunk/block_package length
block_package struct
  block
  txs (coinbase/miner tx included already in block)
  block_size
  cumulative_difficulty
  coins_generated
4-byte chunk/block_package length
block_package struct
[...]

1 files changed, 58 insertions, 7 deletions

diff --git a/src/blockchain_converter/import.h b/src/blockchain_converter/bootstrap_serialization.h
index 632b4c0d9..6fa949353 100644
--- a/src/blockchain_converter/import.h
+++ b/src/blockchain_converter/bootstrap_serialization.h
@@ -28,10 +28,61 @@
 
 #pragma once
 
-// TODO: bounds checking is done before writing to buffer, but buffer size
-// should be a sensible maximum
-#define BUFFER_SIZE 1000000
-#define NUM_BLOCKS_PER_CHUNK 1
-#define STR_LENGTH_OF_INT 9
-#define STR_FORMAT_OF_INT "%09d"
-#define BLOCKCHAIN_RAW "blockchain.raw"
+#include "cryptonote_core/cryptonote_boost_serialization.h"
+#include "cryptonote_core/difficulty.h"
+
+
+namespace cryptonote
+{
+  namespace bootstrap
+  {
+
+    struct file_info
+    {
+      uint8_t  major_version;
+      uint8_t  minor_version;
+      uint32_t header_size;
+
+      BEGIN_SERIALIZE_OBJECT()
+        FIELD(major_version);
+        FIELD(minor_version);
+        VARINT_FIELD(header_size);
+      END_SERIALIZE()
+    };
+
+    struct blocks_info
+    {
+      // block heights of file's first and last blocks, zero-based indexes
+      uint64_t block_first;
+      uint64_t block_last;
+
+      // file position, for directly reading last block
+      uint64_t block_last_pos;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VARINT_FIELD(block_first);
+        VARINT_FIELD(block_last);
+        VARINT_FIELD(block_last_pos);
+      END_SERIALIZE()
+    };
+
+    struct block_package
+    {
+      cryptonote::block block;
+      std::vector<transaction> txs;
+      size_t block_size;
+      difficulty_type cumulative_difficulty;
+      uint64_t coins_generated;
+
+      BEGIN_SERIALIZE()
+        FIELD(block)
+        FIELD(txs)
+        VARINT_FIELD(block_size)
+        VARINT_FIELD(cumulative_difficulty)
+        VARINT_FIELD(coins_generated)
+      END_SERIALIZE()
+    };
+
+  }
+
+}