blockchain_prune: faster

on my anecdotal SSD, goes from about 9 hours to 1h20.

1 files changed, 87 insertions, 41 deletions

diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp
index 4a91cf7cc..a78d7ada9 100644
--- a/src/blockchain_utilities/blockchain_prune.cpp
+++ b/src/blockchain_utilities/blockchain_prune.cpp
@@ -52,9 +52,11 @@ using namespace cryptonote;
 static std::string db_path;
 
 // default to fast:1
-static uint64_t records_per_sync = 128;
+static uint64_t records_per_sync = 16 * 65536;
 static const size_t slack = 512 * 1024 * 1024;
 
+static std::vector<bool> is_v1;
+
 static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name)
 {
   std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string());
@@ -89,6 +91,14 @@ static void close(MDB_env *env)
   mdb_env_close(env);
 }
 
+static void mark_v1_tx(const MDB_val &k, const MDB_val &v)
+{
+  const uint64_t tx_id = *(const uint64_t*)k.mv_data;
+  if (tx_id >= is_v1.size())
+    is_v1.resize(tx_id + 1, false);
+  is_v1[tx_id] = cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size});
+}
+
 static void add_size(MDB_env *env, uint64_t bytes)
 {
   try
@@ -136,7 +146,7 @@ static void check_resize(MDB_env *env, size_t bytes)
     add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize);
 }
 
-static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes)
+static bool resize_point(size_t &nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes)
 {
   if (nrecords % records_per_sync && bytes <= slack / 2)
     return false;
@@ -146,10 +156,11 @@ static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &b
   dbr = mdb_txn_begin(env, NULL, 0, txn);
   if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
   bytes = 0;
+  nrecords = 0;
   return true;
 }
 
-static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0)
+static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0, void (*f)(const MDB_val&, const MDB_val&) = 0)
 {
   MDB_dbi dbi0, dbi1;
   MDB_txn *txn0, *txn1;
@@ -200,6 +211,11 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
   dbr = mdb_cursor_open(txn1, dbi1, &cur1);
   if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
 
+  if (flags & MDB_DUPSORT)
+    putflags |= MDB_APPENDDUP;
+  else
+    putflags |= MDB_APPEND;
+
   MDB_val k;
   MDB_val v;
   MDB_cursor_op op = MDB_FIRST;
@@ -214,7 +230,8 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
       throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret)));
 
     bytes += k.mv_size + v.mv_size;
-    if (resize_point(++nrecords, env1, &txn1, bytes))
+    ++nrecords;
+    if (resize_point(nrecords, env1, &txn1, bytes))
     {
       dbr = mdb_cursor_open(txn1, dbi1, &cur1);
       if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
@@ -223,6 +240,9 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
     ret = mdb_cursor_put(cur1, &k, &v, putflags);
     if (ret)
       throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret)));
+
+    if (f)
+      (*f)(k, v);
   }
 
   mdb_cursor_close(cur1);
@@ -235,17 +255,6 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
   mdb_dbi_close(env0, dbi0);
 }
 
-static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id)
-{
-  MDB_val v;
-  int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET);
-  if (ret)
-    throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret)));
-  if (v.mv_size == 0)
-    throw std::runtime_error("Invalid transaction pruned data");
-  return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size});
-}
-
 static void prune(MDB_env *env0, MDB_env *env1)
 {
   MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties;
@@ -324,7 +333,10 @@ static void prune(MDB_env *env0, MDB_env *env1)
   mdb_dbi_close(env0, dbi0_blocks);
   const uint64_t blockchain_height = stats.ms_entries;
   size_t nrecords = 0, bytes = 0;
+  std::vector<bool> prunable_needed;
 
+  // go through all txes tx indices, recording which ones should have their prunable part retained
+  MINFO("Marking prunable txes");
   MDB_cursor_op op = MDB_FIRST;
   while (1)
   {
@@ -336,7 +348,8 @@ static void prune(MDB_env *env0, MDB_env *env1)
 
     const txindex *ti = (const txindex*)v.mv_data;
     const uint64_t block_height = ti->data.block_id;
-    MDB_val_set(kk, ti->data.tx_id);
+    const uint64_t tx_id = ti->data.tx_id;
+    MDB_val_set(kk, tx_id);
     if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
     {
       MDEBUG(block_height << "/" << blockchain_height << " is in tip");
@@ -344,22 +357,23 @@ static void prune(MDB_env *env0, MDB_env *env1)
       dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0);
       if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr)));
       bytes += kk.mv_size + vv.mv_size;
-    }
-    if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk))
-    {
-      MDB_val vv;
-      dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET);
-      if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr)));
-      bytes += kk.mv_size + vv.mv_size;
-      if (resize_point(++nrecords, env1, &txn1, bytes))
+
+      ++nrecords;
+      if (resize_point(nrecords, env1, &txn1, bytes))
       {
         dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
         if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
         dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
         if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
       }
-      dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0);
-      if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr)));
+    }
+    if (tx_id >= is_v1.size())
+      throw std::runtime_error("tx_id out of range of is_v1 vector");
+    if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1[tx_id])
+    {
+      if (tx_id >= prunable_needed.size())
+        prunable_needed.resize(tx_id + 1, false);
+      prunable_needed[tx_id] = true;
     }
     else
     {
@@ -367,6 +381,37 @@ static void prune(MDB_env *env0, MDB_env *env1)
     }
   }
 
+  // go through prunable parts, carrying over those we need
+  MINFO("Copying retained prunable data");
+  op = MDB_FIRST;
+  while (1)
+  {
+    int ret = mdb_cursor_get(cur0_txs_prunable, &k, &v, op);
+    op = MDB_NEXT;
+    if (ret == MDB_NOTFOUND)
+      break;
+    if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret)));
+
+    const uint64_t tx_id = *(const uint64_t*)k.mv_data;
+    if (tx_id >= prunable_needed.size())
+      throw std::runtime_error("tx_id out of range of prunable_needed vector");
+    if (prunable_needed[tx_id])
+    {
+      dbr = mdb_cursor_put(cur1_txs_prunable, &k, &v, MDB_APPEND);
+      if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr)));
+
+      bytes += k.mv_size + v.mv_size;
+      ++nrecords;
+      if (resize_point(nrecords, env1, &txn1, bytes))
+      {
+        dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
+        if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+        dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
+        if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+      }
+    }
+  }
+
   mdb_cursor_close(cur1_txs_prunable_tip);
   mdb_cursor_close(cur1_txs_prunable);
   mdb_cursor_close(cur0_txs_prunable);
@@ -419,7 +464,7 @@ static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags)
     else if(options[0] == "fastest")
     {
       db_flags = DBF_FASTEST;
-      records_per_sync = 1000; // default to fastest:async:1000
+      // default to fastest:async:N
     }
     else
       return false;
@@ -455,7 +500,7 @@ int main(int argc, char* argv[])
   const command_line::arg_descriptor<std::string> arg_db_sync_mode = {
     "db-sync-mode"
   , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]."
-  , "fast:1000"
+  , "fast:" + std::to_string(records_per_sync)
   };
   const command_line::arg_descriptor<bool> arg_copy_pruned_database  = {"copy-pruned-database",  "Copy database anyway if already pruned"};
 
@@ -601,26 +646,27 @@ int main(int argc, char* argv[])
   MDB_env *env0 = NULL, *env1 = NULL;
   open(env0, paths[0], db_flags, true);
   open(env1, paths[1], db_flags, false);
-  copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND);
-  copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
+  copy_table(env0, env1, "blocks", MDB_INTEGERKEY, 0);
+  copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64);
   copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
   //copy_table(env0, env1, "txs", MDB_INTEGERKEY);
-  copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND);
-  copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND);
+  copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, 0, NULL, &mark_v1_tx);
+  copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0);
   // not copied: prunable, prunable_tip
   copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
-  copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND);
-  copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
-  copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
-  copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
-  copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
-  copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
-  copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND);
+  copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, 0);
+  copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64);
+  copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64);
+  copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
+  copy_table(env0, env1, "txpool_meta", 0, 0, BlockchainLMDB::compare_hash32);
+  copy_table(env0, env1, "txpool_blob", 0, 0, BlockchainLMDB::compare_hash32);
+  copy_table(env0, env1, "alt_blocks", 0, 0, BlockchainLMDB::compare_hash32);
+  copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, 0);
   copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string);
   if (already_pruned)
   {
-    copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64);
-    copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64);
+    copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, 0, BlockchainLMDB::compare_uint64);
+    copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64);
   }
   else
   {