17 files changed, 960 insertions, 251 deletions
diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt
index ddf575c29..0ba7ee86c 100644
--- a/src/blockchain_utilities/CMakeLists.txt
+++ b/src/blockchain_utilities/CMakeLists.txt
@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #
@@ -92,6 +92,17 @@ monero_private_headers(blockchain_prune_known_spent_data
 
 
 
+set(blockchain_prune_sources
+  blockchain_prune.cpp
+  )
+
+set(blockchain_prune_private_headers)
+
+monero_private_headers(blockchain_prune
+	  ${blockchain_prune_private_headers})
+
+
+
 set(blockchain_ancestry_sources
   blockchain_ancestry.cpp
   )
@@ -298,3 +309,25 @@ set_property(TARGET blockchain_prune_known_spent_data
 	PROPERTY
 	OUTPUT_NAME "monero-blockchain-prune-known-spent-data")
 install(TARGETS blockchain_prune_known_spent_data DESTINATION bin)
+
+monero_add_executable(blockchain_prune
+  ${blockchain_prune_sources}
+  ${blockchain_prune_private_headers})
+
+set_property(TARGET blockchain_prune
+	PROPERTY
+	OUTPUT_NAME "monero-blockchain-prune")
+install(TARGETS blockchain_prune DESTINATION bin)
+
+target_link_libraries(blockchain_prune
+  PRIVATE
+    cryptonote_core
+    blockchain_db
+    p2p
+    version
+    epee
+    ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_SYSTEM_LIBRARY}
+    ${Boost_THREAD_LIBRARY}
+    ${CMAKE_THREAD_LIBS_INIT}
+    ${EXTRA_LIBRARIES})
diff --git a/src/blockchain_utilities/README.md b/src/blockchain_utilities/README.md
index 5d968cd75..ad5963f27 100644
--- a/src/blockchain_utilities/README.md
+++ b/src/blockchain_utilities/README.md
@@ -1,6 +1,6 @@
 # Monero Blockchain Utilities
 
-Copyright (c) 2014-2018, The Monero Project
+Copyright (c) 2014-2019, The Monero Project
 
 ## Introduction
 
diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp
index e01a8892c..a6ee0573f 100644
--- a/src/blockchain_utilities/blockchain_ancestry.cpp
+++ b/src/blockchain_utilities/blockchain_ancestry.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
@@ -51,6 +51,8 @@ using namespace epee;
 using namespace cryptonote;
 
 static bool stop_requested = false;
+static uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0;
+static bool opt_cache_outputs = false, opt_cache_txes = false, opt_cache_blocks = false;
 
 struct ancestor
 {
@@ -137,6 +139,8 @@ struct ancestry_state_t
   std::unordered_map<crypto::hash, ::tx_data_t> tx_cache;
   std::vector<cryptonote::block> block_cache;
 
+  ancestry_state_t(): height(0) {}
+
   template <typename t_archive> void serialize(t_archive &a, const unsigned int ver)
   {
     a & height;
@@ -219,6 +223,113 @@ static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto
   return i->second;
 }
 
+static bool get_block_from_height(ancestry_state_t &state, BlockchainDB *db, uint64_t height, cryptonote::block &b)
+{
+  ++total_blocks;
+  if (state.block_cache.size() > height && !state.block_cache[height].miner_tx.vin.empty())
+  {
+    ++cached_blocks;
+    b = state.block_cache[height];
+    return true;
+  }
+  cryptonote::blobdata bd = db->get_block_blob_from_height(height);
+  if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
+  {
+    LOG_PRINT_L0("Bad block from db");
+    return false;
+  }
+  if (opt_cache_blocks)
+  {
+    state.block_cache.resize(height + 1);
+    state.block_cache[height] = b;
+  }
+  return true;
+}
+
+static bool get_transaction(ancestry_state_t &state, BlockchainDB *db, const crypto::hash &txid, ::tx_data_t &tx_data)
+{
+  std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid);
+  ++total_txes;
+  if (i != state.tx_cache.end())
+  {
+    ++cached_txes;
+    tx_data = i->second;
+    return true;
+  }
+
+  cryptonote::blobdata bd;
+  if (!db->get_pruned_tx_blob(txid, bd))
+  {
+    LOG_PRINT_L0("Failed to get txid " << txid << " from db");
+    return false;
+  }
+  cryptonote::transaction tx;
+  if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
+  {
+    LOG_PRINT_L0("Bad tx: " << txid);
+    return false;
+  }
+  tx_data = ::tx_data_t(tx);
+  if (opt_cache_txes)
+    state.tx_cache.insert(std::make_pair(txid, tx_data));
+  return true;
+}
+
+static bool get_output_txid(ancestry_state_t &state, BlockchainDB *db, uint64_t amount, uint64_t offset, crypto::hash &txid)
+{
+  ++total_outputs;
+  std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset});
+  if (i != state.output_cache.end())
+  {
+    ++cached_outputs;
+    txid = i->second;
+    return true;
+  }
+
+  const output_data_t od = db->get_output_key(amount, offset, false);
+  cryptonote::block b;
+  if (!get_block_from_height(state, db, od.height, b))
+    return false;
+
+  for (size_t out = 0; out < b.miner_tx.vout.size(); ++out)
+  {
+    if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key))
+    {
+      const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target);
+      if (txout.key == od.pubkey)
+      {
+        txid = cryptonote::get_transaction_hash(b.miner_tx);
+        if (opt_cache_outputs)
+          state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid));
+        return true;
+      }
+    }
+    else
+    {
+      LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx));
+      return false;
+    }
+  }
+  for (const crypto::hash &block_txid: b.tx_hashes)
+  {
+    ::tx_data_t tx_data3;
+    if (!get_transaction(state, db, block_txid, tx_data3))
+      return false;
+
+    for (size_t out = 0; out < tx_data3.vout.size(); ++out)
+    {
+      if (tx_data3.vout[out] == od.pubkey)
+      {
+        txid = block_txid;
+        if (opt_cache_outputs)
+          state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid));
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 int main(int argc, char* argv[])
 {
   TRY_ENTRY();
@@ -243,12 +354,13 @@ int main(int argc, char* argv[])
     "database", available_dbs.c_str(), default_db_type
   };
   const command_line::arg_descriptor<std::string> arg_txid  = {"txid", "Get ancestry for this txid", ""};
+  const command_line::arg_descriptor<std::string> arg_output  = {"output", "Get ancestry for this output (amount/offset format)", ""};
   const command_line::arg_descriptor<uint64_t> arg_height  = {"height", "Get ancestry for all txes at this height", 0};
-  const command_line::arg_descriptor<bool> arg_all  = {"all", "Include the whole chain", false};
+  const command_line::arg_descriptor<bool> arg_refresh  = {"refresh", "Refresh the whole chain first", false};
   const command_line::arg_descriptor<bool> arg_cache_outputs  = {"cache-outputs", "Cache outputs (memory hungry)", false};
   const command_line::arg_descriptor<bool> arg_cache_txes  = {"cache-txes", "Cache txes (memory hungry)", false};
   const command_line::arg_descriptor<bool> arg_cache_blocks  = {"cache-blocks", "Cache blocks (memory hungry)", false};
-  const command_line::arg_descriptor<bool> arg_include_coinbase  = {"include-coinbase", "Including coinbase tx", false};
+  const command_line::arg_descriptor<bool> arg_include_coinbase  = {"include-coinbase", "Including coinbase tx in per height average", false};
   const command_line::arg_descriptor<bool> arg_show_cache_stats  = {"show-cache-stats", "Show cache statistics", false};
 
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
@@ -257,8 +369,9 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_cmd_sett, arg_log_level);
   command_line::add_arg(desc_cmd_sett, arg_database);
   command_line::add_arg(desc_cmd_sett, arg_txid);
+  command_line::add_arg(desc_cmd_sett, arg_output);
   command_line::add_arg(desc_cmd_sett, arg_height);
-  command_line::add_arg(desc_cmd_sett, arg_all);
+  command_line::add_arg(desc_cmd_sett, arg_refresh);
   command_line::add_arg(desc_cmd_sett, arg_cache_outputs);
   command_line::add_arg(desc_cmd_sett, arg_cache_txes);
   command_line::add_arg(desc_cmd_sett, arg_cache_blocks);
@@ -300,20 +413,22 @@ int main(int argc, char* argv[])
   bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on);
   network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
   std::string opt_txid_string = command_line::get_arg(vm, arg_txid);
+  std::string opt_output_string = command_line::get_arg(vm, arg_output);
   uint64_t opt_height = command_line::get_arg(vm, arg_height);
-  bool opt_all = command_line::get_arg(vm, arg_all);
-  bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs);
-  bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes);
-  bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks);
+  bool opt_refresh = command_line::get_arg(vm, arg_refresh);
+  opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs);
+  opt_cache_txes = command_line::get_arg(vm, arg_cache_txes);
+  opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks);
   bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase);
   bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats);
 
-  if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1)
+  if ((!opt_txid_string.empty()) + !!opt_height + !opt_output_string.empty() > 1)
   {
-    std::cerr << "Only one of --txid, --height and --all can be given" << std::endl;
+    std::cerr << "Only one of --txid, --height, --output can be given" << std::endl;
     return 1;
   }
   crypto::hash opt_txid = crypto::null_hash;
+  uint64_t output_amount = 0, output_offset = 0;
   if (!opt_txid_string.empty())
   {
     if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid))
@@ -322,6 +437,14 @@ int main(int argc, char* argv[])
       return 1;
     }
   }
+  else if (!opt_output_string.empty())
+  {
+    if (sscanf(opt_output_string.c_str(), "%" SCNu64 "/%" SCNu64, &output_amount, &output_offset) != 2)
+    {
+      std::cerr << "Invalid output" << std::endl;
+      return 1;
+    }
+  }
 
   std::string db_type = command_line::get_arg(vm, arg_database);
   if (!cryptonote::blockchain_valid_db_type(db_type))
@@ -372,37 +495,36 @@ int main(int argc, char* argv[])
 
   std::vector<crypto::hash> start_txids;
 
-  // forward method
-  if (opt_all)
-  {
-    uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0;
-    ancestry_state_t state;
+  ancestry_state_t state;
 
-    const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string();
-    LOG_PRINT_L0("Loading state data from " << state_file_path);
-    std::ifstream state_data_in;
-    state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in);
-    if (!state_data_in.fail())
+  const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string();
+  LOG_PRINT_L0("Loading state data from " << state_file_path);
+  std::ifstream state_data_in;
+  state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in);
+  if (!state_data_in.fail())
+  {
+    try
     {
-      try
-      {
-        boost::archive::portable_binary_iarchive a(state_data_in);
-        a >> state;
-      }
-      catch (const std::exception &e)
-      {
-        MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch");
-        state = ancestry_state_t();
-      }
-      state_data_in.close();
+      boost::archive::portable_binary_iarchive a(state_data_in);
+      a >> state;
+    }
+    catch (const std::exception &e)
+    {
+      MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch");
+      state = ancestry_state_t();
     }
+    state_data_in.close();
+  }
 
-    tools::signal_handler::install([](int type) {
-      stop_requested = true;
-    });
+  tools::signal_handler::install([](int type) {
+    stop_requested = true;
+  });
 
+  // forward method
+  const uint64_t db_height = db->height();
+  if (opt_refresh)
+  {
     MINFO("Starting from height " << state.height);
-    const uint64_t db_height = db->height();
     state.block_cache.reserve(db_height);
     for (uint64_t h = state.height; h < db_height; ++h)
     {
@@ -464,113 +586,20 @@ int main(int argc, char* argv[])
         {
           for (size_t ring = 0; ring < tx_data.vin.size(); ++ring)
           {
-            if (1)
+            const uint64_t amount = tx_data.vin[ring].first;
+            const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second;
+            for (uint64_t offset: absolute_offsets)
             {
-              const uint64_t amount = tx_data.vin[ring].first;
-              const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second;
-              for (uint64_t offset: absolute_offsets)
+              add_ancestry(state.ancestry, txid, ancestor{amount, offset});
+              // find the tx which created this output
+              bool found = false;
+              crypto::hash output_txid;
+              if (!get_output_txid(state, db, amount, offset, output_txid))
               {
-                const output_data_t od = db->get_output_key(amount, offset);
-                add_ancestry(state.ancestry, txid, ancestor{amount, offset});
-                cryptonote::block b;
-                ++total_blocks;
-                if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty())
-                {
-                  ++cached_blocks;
-                  b = state.block_cache[od.height];
-                }
-                else
-                {
-                  cryptonote::blobdata bd = db->get_block_blob_from_height(od.height);
-                  if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
-                  {
-                    LOG_PRINT_L0("Bad block from db");
-                    return 1;
-                  }
-                  if (opt_cache_blocks)
-                  {
-                    state.block_cache.resize(od.height + 1);
-                    state.block_cache[od.height] = b;
-                  }
-                }
-                // find the tx which created this output
-                bool found = false;
-                std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset});
-                ++total_outputs;
-                if (i != state.output_cache.end())
-                {
-                  ++cached_outputs;
-                  add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second));
-                  found = true;
-                }
-                else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out)
-                {
-                  if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key))
-                  {
-                    const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target);
-                    if (txout.key == od.pubkey)
-                    {
-                      found = true;
-                      add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx)));
-                      if (opt_cache_outputs)
-                        state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx)));
-                      break;
-                    }
-                  }
-                  else
-                  {
-                    LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx));
-                    return 1;
-                  }
-                }
-                for (const crypto::hash &block_txid: b.tx_hashes)
-                {
-                  if (found)
-                    break;
-                  ::tx_data_t tx_data2;
-                  std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid);
-                  ++total_txes;
-                  if (i != state.tx_cache.end())
-                  {
-                    ++cached_txes;
-                    tx_data2 = i->second;
-                  }
-                  else
-                  {
-                    cryptonote::blobdata bd;
-                    if (!db->get_pruned_tx_blob(block_txid, bd))
-                    {
-                      LOG_PRINT_L0("Failed to get txid " << block_txid << " from db");
-                      return 1;
-                    }
-                    cryptonote::transaction tx;
-                    if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
-                    {
-                      LOG_PRINT_L0("Bad tx: " << block_txid);
-                      return 1;
-                    }
-                    tx_data2 = ::tx_data_t(tx);
-                    if (opt_cache_txes)
-                      state.tx_cache.insert(std::make_pair(block_txid, tx_data2));
-                  }
-                  for (size_t out = 0; out < tx_data2.vout.size(); ++out)
-                  {
-                    if (tx_data2.vout[out] == od.pubkey)
-                    {
-                      found = true;
-                      add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid));
-                      if (opt_cache_outputs)
-                        state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid));
-                      break;
-                    }
-                  }
-                }
-                if (!found)
-                {
-                  LOG_PRINT_L0("Output originating transaction not found");
-                  return 1;
-                }
+                LOG_PRINT_L0("Output originating transaction not found");
+                return 1;
               }
+              add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid));
             }
           }
         }
@@ -581,10 +610,6 @@ int main(int argc, char* argv[])
       if (!txids.empty())
       {
         std::string stats_msg;
-        if (opt_show_cache_stats)
-          stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes)
-              + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs "
-              + std::to_string(cached_outputs*100./total_outputs);
         MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg);
       }
       state.height = h;
@@ -608,14 +633,30 @@ int main(int argc, char* argv[])
       }
       state_data_out.close();
     }
-
-    goto done;
+  }
+  else
+  {
+    if (state.height < db_height)
+    {
+      MWARNING("The state file is only built up to height " << state.height << ", but the blockchain reached height " << db_height);
+      MWARNING("You may want to run with --refresh if you want to get ancestry for newer data");
+    }
   }
 
   if (!opt_txid_string.empty())
   {
     start_txids.push_back(opt_txid);
   }
+  else if (!opt_output_string.empty())
+  {
+    crypto::hash txid;
+    if (!get_output_txid(state, db, output_amount, output_offset, txid))
+    {
+      LOG_PRINT_L0("Output not found in db");
+      return 1;
+    }
+    start_txids.push_back(txid);
+  }
   else
   {
     const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height);
@@ -648,108 +689,40 @@ int main(int argc, char* argv[])
       const crypto::hash txid = txids.front();
       txids.pop_front();
 
-      cryptonote::blobdata bd;
-      if (!db->get_pruned_tx_blob(txid, bd))
-      {
-        LOG_PRINT_L0("Failed to get txid " << txid << " from db");
-        return 1;
-      }
-      cryptonote::transaction tx;
-      if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
-      {
-        LOG_PRINT_L0("Bad tx: " << txid);
+      if (stop_requested)
+        goto done;
+
+      ::tx_data_t tx_data2;
+      if (!get_transaction(state, db, txid, tx_data2))
         return 1;
-      }
-      const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen);
+
+      const bool coinbase = tx_data2.coinbase;
       if (coinbase)
         continue;
 
-      for (size_t ring = 0; ring < tx.vin.size(); ++ring)
+      for (size_t ring = 0; ring < tx_data2.vin.size(); ++ring)
       {
-        if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key))
         {
-          const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]);
-          const uint64_t amount = txin.amount;
-          auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets);
+          const uint64_t amount = tx_data2.vin[ring].first;
+          auto absolute_offsets = tx_data2.vin[ring].second;
           for (uint64_t offset: absolute_offsets)
           {
             add_ancestor(ancestry, amount, offset);
-            const output_data_t od = db->get_output_key(amount, offset);
-            bd = db->get_block_blob_from_height(od.height);
-            cryptonote::block b;
-            if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
-            {
-              LOG_PRINT_L0("Bad block from db");
-              return 1;
-            }
+
             // find the tx which created this output
             bool found = false;
-            for (size_t out = 0; out < b.miner_tx.vout.size(); ++out)
-            {
-              if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key))
-              {
-                const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target);
-                if (txout.key == od.pubkey)
-                {
-                  found = true;
-                  txids.push_back(cryptonote::get_transaction_hash(b.miner_tx));
-                  MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx));
-                  break;
-                }
-              }
-              else
-              {
-                LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx));
-                return 1;
-              }
-            }
-            for (const crypto::hash &block_txid: b.tx_hashes)
-            {
-              if (found)
-                break;
-              if (!db->get_pruned_tx_blob(block_txid, bd))
-              {
-                LOG_PRINT_L0("Failed to get txid " << block_txid << " from db");
-                return 1;
-              }
-              cryptonote::transaction tx2;
-              if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2))
-              {
-                LOG_PRINT_L0("Bad tx: " << block_txid);
-                return 1;
-              }
-              for (size_t out = 0; out < tx2.vout.size(); ++out)
-              {
-                if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key))
-                {
-                  const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target);
-                  if (txout.key == od.pubkey)
-                  {
-                    found = true;
-                    txids.push_back(block_txid);
-                    MDEBUG("adding txid: " << block_txid);
-                    break;
-                  }
-                }
-                else
-                {
-                  LOG_PRINT_L0("Bad vout type in txid " << block_txid);
-                  return 1;
-                }
-              }
-            }
-            if (!found)
+            crypto::hash output_txid;
+            if (!get_output_txid(state, db, amount, offset, output_txid))
             {
               LOG_PRINT_L0("Output originating transaction not found");
               return 1;
             }
+
+            add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid));
+            txids.push_back(output_txid);
+            MDEBUG("adding txid: " << output_txid);
           }
         }
-        else
-        {
-          LOG_PRINT_L0("Bad vin type in txid " << txid);
-          return 1;
-        }
       }
     }
 
@@ -762,6 +735,13 @@ int main(int argc, char* argv[])
 
 done:
   core_storage->deinit();
+
+  if (opt_show_cache_stats)
+  MINFO("cache: txes " << std::to_string(cached_txes*100./total_txes)
+        << "%, blocks " << std::to_string(cached_blocks*100./total_blocks)
+        << "%, outputs " << std::to_string(cached_outputs*100./total_outputs)
+        << "%");
+
   return 0;
 
   CATCH_ENTRY("Depth query error", 1);
diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp
index 73819bd25..6ff184041 100644
--- a/src/blockchain_utilities/blockchain_blackball.cpp
+++ b/src/blockchain_utilities/blockchain_blackball.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
@@ -1131,7 +1131,7 @@ int main(int argc, char* argv[])
     return 1;
   }
 
-  mlog_configure(mlog_get_default_log_path("monero-blockchain-find-spent-outputs.log"), true);
+  mlog_configure(mlog_get_default_log_path("monero-blockchain-mark-spent-outputs.log"), true);
   if (!command_line::is_arg_defaulted(vm, arg_log_level))
     mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
   else
diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp
index 8060b0de4..8be83ee67 100644
--- a/src/blockchain_utilities/blockchain_depth.cpp
+++ b/src/blockchain_utilities/blockchain_depth.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp
index 5a49f3478..fa1243c1f 100644
--- a/src/blockchain_utilities/blockchain_export.cpp
+++ b/src/blockchain_utilities/blockchain_export.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
@@ -177,6 +177,12 @@ int main(int argc, char* argv[])
   }
   r = core_storage->init(db, opt_testnet ? cryptonote::TESTNET : opt_stagenet ? cryptonote::STAGENET : cryptonote::MAINNET);
 
+  if (core_storage->get_blockchain_pruning_seed())
+  {
+    LOG_PRINT_L0("Blockchain is pruned, cannot export");
+    return 1;
+  }
+
   CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
   LOG_PRINT_L0("Source blockchain storage initialized OK");
   LOG_PRINT_L0("Exporting blockchain raw data...");
diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp
index 2a8a6f494..e4efdc3cb 100644
--- a/src/blockchain_utilities/blockchain_import.cpp
+++ b/src/blockchain_utilities/blockchain_import.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
@@ -193,8 +193,16 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
   }
   core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes);
 
-  core.prepare_handle_incoming_blocks(blocks);
+  std::vector<block> pblocks;
+  core.prepare_handle_incoming_blocks(blocks, pblocks);
+  if (!pblocks.empty() && pblocks.size() != blocks.size())
+  {
+    MERROR("Unexpected parsed blocks size");
+    core.cleanup_handle_incoming_blocks();
+    return 1;
+  }
 
+  size_t blockidx = 0;
   for(const block_complete_entry& block_entry: blocks)
   {
     // process transactions
@@ -215,7 +223,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
 
     block_verification_context bvc = boost::value_initialized<block_verification_context>();
 
-    core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block
+    core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx++], bvc, false); // <--- process block
 
     if(bvc.m_verifivation_failed)
     {
@@ -455,7 +463,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
         }
         else
         {
-          std::vector<transaction> txs;
+          std::vector<std::pair<transaction, blobdata>> txs;
           std::vector<transaction> archived_txs;
 
           archived_txs = bp.txs;
@@ -472,7 +480,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
             // because add_block() calls
             // add_transaction(blk_hash, blk.miner_tx) first, and
             // then a for loop for the transactions in txs.
-            txs.push_back(tx);
+            txs.push_back(std::make_pair(tx, tx_to_blob(tx)));
           }
 
           size_t block_weight;
@@ -485,7 +493,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
 
           try
           {
-            core.get_blockchain_storage().get_db().add_block(b, block_weight, cumulative_difficulty, coins_generated, txs);
+            uint64_t long_term_block_weight = core.get_blockchain_storage().get_next_long_term_block_weight(block_weight);
+            core.get_blockchain_storage().get_db().add_block(std::make_pair(b, block_to_blob(b)), block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs);
           }
           catch (const std::exception& e)
           {
diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp
new file mode 100644
index 000000000..36080aade
--- /dev/null
+++ b/src/blockchain_utilities/blockchain_prune.cpp
@@ -0,0 +1,681 @@
+// Copyright (c) 2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <array>
+#include <lmdb.h>
+#include <boost/algorithm/string.hpp>
+#include "common/command_line.h"
+#include "common/pruning.h"
+#include "cryptonote_core/cryptonote_core.h"
+#include "cryptonote_core/blockchain.h"
+#include "blockchain_db/blockchain_db.h"
+#include "blockchain_db/lmdb/db_lmdb.h"
+#include "blockchain_db/db_types.h"
+#include "version.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
+
+#define MDB_val_set(var, val)   MDB_val var = {sizeof(val), (void *)&val}
+
+namespace po = boost::program_options;
+using namespace epee;
+using namespace cryptonote;
+
+static std::string db_path;
+
+// default to fast:1
+static uint64_t records_per_sync = 128;
+static const size_t slack = 512 * 1024 * 1024;
+
+static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name)
+{
+  std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string());
+  if (ec)
+    MERROR("Error renaming " << replacement_name << " to " << replaced_name << ": " << ec.message());
+  return ec;
+}
+
+static void open(MDB_env *&env, const boost::filesystem::path &path, uint64_t db_flags, bool readonly)
+{
+  int dbr;
+  int flags = 0;
+
+  if (db_flags & DBF_FAST)
+    flags |= MDB_NOSYNC;
+  if (db_flags & DBF_FASTEST)
+    flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC;
+  if (readonly)
+    flags |= MDB_RDONLY;
+
+  dbr = mdb_env_create(&env);
+  if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_set_maxdbs(env, 32);
+  if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_open(env, path.string().c_str(), flags, 0664);
+  if (dbr) throw std::runtime_error("Failed to open database file '"
+      + path.string() + "': " + std::string(mdb_strerror(dbr)));
+}
+
+static void close(MDB_env *env)
+{
+  mdb_env_close(env);
+}
+
+static void add_size(MDB_env *env, uint64_t bytes)
+{
+  try
+  {
+    boost::filesystem::path path(db_path);
+    boost::filesystem::space_info si = boost::filesystem::space(path);
+    if(si.available < bytes)
+    {
+      MERROR("!! WARNING: Insufficient free space to extend database !!: " <<
+          (si.available >> 20L) << " MB available, " << (bytes >> 20L) << " MB needed");
+      return;
+    }
+  }
+  catch(...)
+  {
+    // print something but proceed.
+    MWARNING("Unable to query free disk space.");
+  }
+
+  MDB_envinfo mei;
+  mdb_env_info(env, &mei);
+  MDB_stat mst;
+  mdb_env_stat(env, &mst);
+
+  uint64_t new_mapsize = (uint64_t)mei.me_mapsize + bytes;
+  new_mapsize += (new_mapsize % mst.ms_psize);
+
+  int result = mdb_env_set_mapsize(env, new_mapsize);
+  if (result)
+    throw std::runtime_error("Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " + std::string(mdb_strerror(result)));
+
+  MGINFO("LMDB Mapsize increased." << "  Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
+}
+
+static void check_resize(MDB_env *env, size_t bytes)
+{
+  MDB_envinfo mei;
+  MDB_stat mst;
+
+  mdb_env_info(env, &mei);
+  mdb_env_stat(env, &mst);
+
+  uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
+  if (size_used + bytes + slack >= mei.me_mapsize)
+    add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize);
+}
+
+static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes)
+{
+  if (nrecords % records_per_sync && bytes <= slack / 2)
+    return false;
+  int dbr = mdb_txn_commit(*txn);
+  if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
+  check_resize(env, bytes);
+  dbr = mdb_txn_begin(env, NULL, 0, txn);
+  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  bytes = 0;
+  return true;
+}
+
+static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0)
+{
+  MDB_dbi dbi0, dbi1;
+  MDB_txn *txn0, *txn1;
+  MDB_cursor *cur0, *cur1;
+  bool tx_active0 = false, tx_active1 = false;
+  int dbr;
+
+  MINFO("Copying " << table);
+
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){
+    if (tx_active1) mdb_txn_abort(txn1);
+    if (tx_active0) mdb_txn_abort(txn0);
+  });
+
+  dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0);
+  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  tx_active0 = true;
+  dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
+  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  tx_active1 = true;
+
+  dbr = mdb_dbi_open(txn0, table, flags, &dbi0);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  if (cmp)
+    ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn0, dbi0, cmp);
+
+  dbr = mdb_dbi_open(txn1, table, flags, &dbi1);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  if (cmp)
+    ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn1, dbi1, cmp);
+
+  dbr = mdb_txn_commit(txn1);
+  if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
+  tx_active1 = false;
+  MDB_stat stats;
+  dbr = mdb_env_stat(env0, &stats);
+  if (dbr) throw std::runtime_error("Failed to stat " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr)));
+  check_resize(env1, (stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) * stats.ms_psize);
+  dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
+  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  tx_active1 = true;
+
+  dbr = mdb_drop(txn1, dbi1, 0);
+  if (dbr) throw std::runtime_error("Failed to empty " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_cursor_open(txn0, dbi0, &cur0);
+  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_cursor_open(txn1, dbi1, &cur1);
+  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+
+  MDB_val k;
+  MDB_val v;
+  MDB_cursor_op op = MDB_FIRST;
+  size_t nrecords = 0, bytes = 0;
+  while (1)
+  {
+    int ret = mdb_cursor_get(cur0, &k, &v, op);
+    op = MDB_NEXT;
+    if (ret == MDB_NOTFOUND)
+      break;
+    if (ret)
+      throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret)));
+
+    bytes += k.mv_size + v.mv_size;
+    if (resize_point(++nrecords, env1, &txn1, bytes))
+    {
+      dbr = mdb_cursor_open(txn1, dbi1, &cur1);
+      if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+    }
+
+    ret = mdb_cursor_put(cur1, &k, &v, putflags);
+    if (ret)
+      throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret)));
+  }
+
+  mdb_cursor_close(cur1);
+  mdb_cursor_close(cur0);
+  mdb_txn_commit(txn1);
+  tx_active1 = false;
+  mdb_txn_commit(txn0);
+  tx_active0 = false;
+  mdb_dbi_close(env1, dbi1);
+  mdb_dbi_close(env0, dbi0);
+}
+
+static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id)
+{
+  MDB_val v;
+  int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET);
+  if (ret)
+    throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret)));
+  if (v.mv_size == 0)
+    throw std::runtime_error("Invalid transaction pruned data");
+  return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size});
+}
+
+static void prune(MDB_env *env0, MDB_env *env1)
+{
+  MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties;
+  MDB_txn *txn0, *txn1;
+  MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable, *cur1_txs_prunable_tip;
+  bool tx_active0 = false, tx_active1 = false;
+  int dbr;
+
+  MGINFO("Creating pruned txs_prunable");
+
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){
+    if (tx_active1) mdb_txn_abort(txn1);
+    if (tx_active0) mdb_txn_abort(txn0);
+  });
+
+  dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0);
+  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  tx_active0 = true;
+  dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
+  if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  tx_active1 = true;
+
+  dbr = mdb_dbi_open(txn0, "txs_pruned", MDB_INTEGERKEY, &dbi0_txs_pruned);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  mdb_set_compare(txn0, dbi0_txs_pruned, BlockchainLMDB::compare_uint64);
+  dbr = mdb_cursor_open(txn0, dbi0_txs_pruned, &cur0_txs_pruned);
+  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_dbi_open(txn0, "txs_prunable", MDB_INTEGERKEY, &dbi0_txs_prunable);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  mdb_set_compare(txn0, dbi0_txs_prunable, BlockchainLMDB::compare_uint64);
+  dbr = mdb_cursor_open(txn0, dbi0_txs_prunable, &cur0_txs_prunable);
+  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_dbi_open(txn0, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi0_tx_indices);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  mdb_set_dupsort(txn0, dbi0_tx_indices, BlockchainLMDB::compare_hash32);
+  dbr = mdb_cursor_open(txn0, dbi0_tx_indices, &cur0_tx_indices);
+  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_dbi_open(txn1, "txs_prunable", MDB_INTEGERKEY, &dbi1_txs_prunable);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  mdb_set_compare(txn1, dbi1_txs_prunable, BlockchainLMDB::compare_uint64);
+  dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
+  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_dbi_open(txn1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi1_txs_prunable_tip);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  mdb_set_dupsort(txn1, dbi1_txs_prunable_tip, BlockchainLMDB::compare_uint64);
+  dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
+  if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_drop(txn1, dbi1_txs_prunable, 0);
+  if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_drop(txn1, dbi1_txs_prunable_tip, 0);
+  if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
+
+  dbr = mdb_dbi_open(txn1, "properties", 0, &dbi1_properties);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+
+  MDB_val k, v;
+  uint32_t pruning_seed = tools::make_pruning_seed(tools::get_random_stripe(), CRYPTONOTE_PRUNING_LOG_STRIPES);
+  static char pruning_seed_key[] = "pruning_seed";
+  k.mv_data = pruning_seed_key;
+  k.mv_size = strlen("pruning_seed") + 1;
+  v.mv_data = (void*)&pruning_seed;
+  v.mv_size = sizeof(pruning_seed);
+  dbr = mdb_put(txn1, dbi1_properties, &k, &v, 0);
+  if (dbr) throw std::runtime_error("Failed to save pruning seed: " + std::string(mdb_strerror(dbr)));
+
+  MDB_stat stats;
+  dbr = mdb_dbi_open(txn0, "blocks", 0, &dbi0_blocks);
+  if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_stat(txn0, dbi0_blocks, &stats);
+  if (dbr) throw std::runtime_error("Failed to query size of blocks: " + std::string(mdb_strerror(dbr)));
+  mdb_dbi_close(env0, dbi0_blocks);
+  const uint64_t blockchain_height = stats.ms_entries;
+  size_t nrecords = 0, bytes = 0;
+
+  MDB_cursor_op op = MDB_FIRST;
+  while (1)
+  {
+    int ret = mdb_cursor_get(cur0_tx_indices, &k, &v, op);
+    op = MDB_NEXT;
+    if (ret == MDB_NOTFOUND)
+      break;
+    if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret)));
+
+    const txindex *ti = (const txindex*)v.mv_data;
+    const uint64_t block_height = ti->data.block_id;
+    MDB_val_set(kk, ti->data.tx_id);
+    if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height)
+    {
+      MDEBUG(block_height << "/" << blockchain_height << " is in tip");
+      MDB_val_set(vv, block_height);
+      dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0);
+      if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr)));
+      bytes += kk.mv_size + vv.mv_size;
+    }
+    if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk))
+    {
+      MDB_val vv;
+      dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET);
+      if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr)));
+      bytes += kk.mv_size + vv.mv_size;
+      if (resize_point(++nrecords, env1, &txn1, bytes))
+      {
+        dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
+        if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+        dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
+        if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
+      }
+      dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0);
+      if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr)));
+    }
+    else
+    {
+      MDEBUG("" << block_height << "/" << blockchain_height << " should be pruned, dropping");
+    }
+  }
+
+  mdb_cursor_close(cur1_txs_prunable_tip);
+  mdb_cursor_close(cur1_txs_prunable);
+  mdb_cursor_close(cur0_txs_prunable);
+  mdb_cursor_close(cur0_txs_pruned);
+  mdb_cursor_close(cur0_tx_indices);
+  mdb_txn_commit(txn1);
+  tx_active1 = false;
+  mdb_txn_commit(txn0);
+  tx_active0 = false;
+  mdb_dbi_close(env1, dbi1_properties);
+  mdb_dbi_close(env1, dbi1_txs_prunable_tip);
+  mdb_dbi_close(env1, dbi1_txs_prunable);
+  mdb_dbi_close(env0, dbi0_txs_prunable);
+  mdb_dbi_close(env0, dbi0_txs_pruned);
+  mdb_dbi_close(env0, dbi0_tx_indices);
+}
+
+static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags)
+{
+  std::vector<std::string> options;
+  boost::trim(db_sync_mode);
+  boost::split(options, db_sync_mode, boost::is_any_of(" :"));
+
+  for(const auto &option : options)
+    MDEBUG("option: " << option);
+
+  // default to fast:async:1
+  uint64_t DEFAULT_FLAGS = DBF_FAST;
+
+  db_flags = 0;
+
+  if(options.size() == 0)
+  {
+    // default to fast:async:1
+    db_flags = DEFAULT_FLAGS;
+  }
+
+  bool safemode = false;
+  if(options.size() >= 1)
+  {
+    if(options[0] == "safe")
+    {
+      safemode = true;
+      db_flags = DBF_SAFE;
+    }
+    else if(options[0] == "fast")
+    {
+      db_flags = DBF_FAST;
+    }
+    else if(options[0] == "fastest")
+    {
+      db_flags = DBF_FASTEST;
+      records_per_sync = 1000; // default to fastest:async:1000
+    }
+    else
+      return false;
+  }
+
+  if(options.size() >= 2 && !safemode)
+  {
+    char *endptr;
+    uint64_t bps = strtoull(options[1].c_str(), &endptr, 0);
+    if (*endptr != '\0')
+      return false;
+    records_per_sync = bps;
+  }
+
+  return true;
+}
+
+int main(int argc, char* argv[])
+{
+  TRY_ENTRY();
+
+  epee::string_tools::set_module_name_and_folder(argv[0]);
+
+  std::string default_db_type = "lmdb";
+
+  std::string available_dbs = cryptonote::blockchain_db_types(", ");
+  available_dbs = "available: " + available_dbs;
+
+  uint32_t log_level = 0;
+
+  tools::on_startup();
+
+  boost::filesystem::path output_file_path;
+
+  po::options_description desc_cmd_only("Command line options");
+  po::options_description desc_cmd_sett("Command line options and settings options");
+  const command_line::arg_descriptor<std::string> arg_log_level  = {"log-level",  "0-4 or categories", ""};
+  const command_line::arg_descriptor<std::string> arg_database = {
+    "database", available_dbs.c_str(), default_db_type
+  };
+  const command_line::arg_descriptor<std::string> arg_db_sync_mode = {
+    "db-sync-mode"
+  , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]."
+  , "fast:1000"
+  };
+  const command_line::arg_descriptor<bool> arg_copy_pruned_database  = {"copy-pruned-database",  "Copy database anyway if already pruned"};
+
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on);
+  command_line::add_arg(desc_cmd_sett, arg_log_level);
+  command_line::add_arg(desc_cmd_sett, arg_database);
+  command_line::add_arg(desc_cmd_sett, arg_db_sync_mode);
+  command_line::add_arg(desc_cmd_sett, arg_copy_pruned_database);
+  command_line::add_arg(desc_cmd_only, command_line::arg_help);
+
+  po::options_description desc_options("Allowed options");
+  desc_options.add(desc_cmd_only).add(desc_cmd_sett);
+
+  po::variables_map vm;
+  bool r = command_line::handle_error_helper(desc_options, [&]()
+  {
+    auto parser = po::command_line_parser(argc, argv).options(desc_options);
+    po::store(parser.run(), vm);
+    po::notify(vm);
+    return true;
+  });
+  if (! r)
+    return 1;
+
+  if (command_line::get_arg(vm, command_line::arg_help))
+  {
+    std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL;
+    std::cout << desc_options << std::endl;
+    return 1;
+  }
+
+  mlog_configure(mlog_get_default_log_path("monero-blockchain-prune.log"), true);
+  if (!command_line::is_arg_defaulted(vm, arg_log_level))
+    mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
+  else
+    mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
+
+  MINFO("Starting...");
+
+  bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
+  bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on);
+  network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
+  bool opt_copy_pruned_database = command_line::get_arg(vm, arg_copy_pruned_database);
+  std::string data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir);
+  while (boost::ends_with(data_dir, "/") || boost::ends_with(data_dir, "\\"))
+    data_dir.pop_back();
+
+  std::string db_type = command_line::get_arg(vm, arg_database);
+  if (!cryptonote::blockchain_valid_db_type(db_type))
+  {
+    MERROR("Invalid database type: " << db_type);
+    return 1;
+  }
+  if (db_type != "lmdb")
+  {
+    MERROR("Unsupported database type: " << db_type << ". Only lmdb is supported");
+    return 1;
+  }
+
+  std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode);
+  uint64_t db_flags = 0;
+  if (!parse_db_sync_mode(db_sync_mode, db_flags))
+  {
+    MERROR("Invalid db sync mode: " << db_sync_mode);
+    return 1;
+  }
+
+  // If we wanted to use the memory pool, we would set up a fake_core.
+
+  // Use Blockchain instead of lower-level BlockchainDB for two reasons:
+  // 1. Blockchain has the init() method for easy setup
+  // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash()
+  //
+  // cannot match blockchain_storage setup above with just one line,
+  // e.g.
+  //   Blockchain* core_storage = new Blockchain(NULL);
+  // because unlike blockchain_storage constructor, which takes a pointer to
+  // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object.
+  MINFO("Initializing source blockchain (BlockchainDB)");
+  std::array<std::unique_ptr<Blockchain>, 2> core_storage;
+  Blockchain *blockchain = NULL;
+  tx_memory_pool m_mempool(*blockchain);
+  boost::filesystem::path paths[2];
+  bool already_pruned = false;
+  for (size_t n = 0; n < core_storage.size(); ++n)
+  {
+    core_storage[n].reset(new Blockchain(m_mempool));
+
+    BlockchainDB* db = new_db(db_type);
+    if (db == NULL)
+    {
+      MERROR("Attempted to use non-existent database type: " << db_type);
+      throw std::runtime_error("Attempting to use non-existent database type");
+    }
+    MDEBUG("database: " << db_type);
+
+    if (n == 1)
+    {
+      paths[1] = boost::filesystem::path(data_dir) / (db->get_db_name() + "-pruned");
+      if (boost::filesystem::exists(paths[1]))
+      {
+        if (!boost::filesystem::is_directory(paths[1]))
+        {
+          MERROR("LMDB needs a directory path, but a file was passed: " << paths[1].string());
+          return 1;
+        }
+      }
+      else
+      {
+        if (!boost::filesystem::create_directories(paths[1]))
+        {
+          MERROR("Failed to create directory: " << paths[1].string());
+          return 1;
+        }
+      }
+      db_path = paths[1].string();
+    }
+    else
+    {
+      paths[0] = boost::filesystem::path(data_dir) / db->get_db_name();
+    }
+
+    MINFO("Loading blockchain from folder " << paths[n] << " ...");
+
+    try
+    {
+      db->open(paths[n].string(), n == 0 ? DBF_RDONLY : 0);
+    }
+    catch (const std::exception& e)
+    {
+      MERROR("Error opening database: " << e.what());
+      return 1;
+    }
+    r = core_storage[n]->init(db, net_type);
+
+    std::string source_dest = n == 0 ? "source" : "pruned";
+    CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize " << source_dest << " blockchain storage");
+    MINFO(source_dest << " blockchain storage initialized OK");
+    if (n == 0 && core_storage[0]->get_blockchain_pruning_seed())
+    {
+      if (!opt_copy_pruned_database)
+      {
+        MERROR("Blockchain is already pruned, use --" << arg_copy_pruned_database.name << " to copy it anyway");
+        return 1;
+      }
+      already_pruned = true;
+    }
+    if (n == 0)
+    {
+      const uint64_t blockchain_height = core_storage[0]->get_current_blockchain_height();
+      const crypto::hash hash = core_storage[0]->get_block_id_by_height(blockchain_height - 1);
+      cryptonote::block block;
+      if (core_storage[0]->get_block_by_hash(hash, block))
+      {
+        if (block.major_version < 10)
+        {
+          time_t now = time(NULL);
+          if (now < 1555286400) // 15 april 2019
+          {
+            MERROR("Pruning before v10 will confuse peers. Wait for v10 first");
+            return 1;
+          }
+        }
+      }
+    }
+  }
+  core_storage[0]->deinit();
+  core_storage[0].reset(NULL);
+  core_storage[1]->deinit();
+  core_storage[1].reset(NULL);
+
+  MINFO("Pruning...");
+  MDB_env *env0 = NULL, *env1 = NULL;
+  open(env0, paths[0], db_flags, true);
+  open(env1, paths[1], db_flags, false);
+  copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND);
+  copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
+  copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
+  //copy_table(env0, env1, "txs", MDB_INTEGERKEY);
+  copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND);
+  copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND);
+  // not copied: prunable, prunable_tip
+  copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
+  copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND);
+  copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
+  copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
+  copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
+  copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
+  copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
+  copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND);
+  copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string);
+  if (already_pruned)
+  {
+    copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64);
+    copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64);
+  }
+  else
+  {
+    prune(env0, env1);
+  }
+  close(env1);
+  close(env0);
+
+  MINFO("Swapping databases, pre-pruning blockchain will be left in " << paths[0].string() + "-old and can be removed if desired");
+  if (replace_file(paths[0].string(), paths[0].string() + "-old") || replace_file(paths[1].string(), paths[0].string()))
+  {
+    MERROR("Blockchain pruned OK, but renaming failed");
+    return 1;
+  }
+
+  MINFO("Blockchain pruned OK");
+  return 0;
+
+  CATCH_ENTRY("Pruning error", 1);
+}
diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp
index f6136c1ba..2d49b6ecd 100644
--- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp
+++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp
index aae8f333b..4cc84bf4a 100644
--- a/src/blockchain_utilities/blockchain_stats.cpp
+++ b/src/blockchain_utilities/blockchain_stats.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp
index 38a0b2648..bd73350b3 100644
--- a/src/blockchain_utilities/blockchain_usage.cpp
+++ b/src/blockchain_utilities/blockchain_usage.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
diff --git a/src/blockchain_utilities/blockchain_utilities.h b/src/blockchain_utilities/blockchain_utilities.h
index e690305c4..78487b995 100644
--- a/src/blockchain_utilities/blockchain_utilities.h
+++ b/src/blockchain_utilities/blockchain_utilities.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
diff --git a/src/blockchain_utilities/blocksdat_file.cpp b/src/blockchain_utilities/blocksdat_file.cpp
index 45ef33acb..f56ff5f94 100644
--- a/src/blockchain_utilities/blocksdat_file.cpp
+++ b/src/blockchain_utilities/blocksdat_file.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h
index 70a1f30a7..315713424 100644
--- a/src/blockchain_utilities/blocksdat_file.h
+++ b/src/blockchain_utilities/blocksdat_file.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp
index a8c46d661..fb9a24f5d 100644
--- a/src/blockchain_utilities/bootstrap_file.cpp
+++ b/src/blockchain_utilities/bootstrap_file.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h
index 187db0938..5fb2cf366 100644
--- a/src/blockchain_utilities/bootstrap_file.h
+++ b/src/blockchain_utilities/bootstrap_file.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h
index 278a7b40f..554c6d56e 100644
--- a/src/blockchain_utilities/bootstrap_serialization.h
+++ b/src/blockchain_utilities/bootstrap_serialization.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //