Add functions to iterate through blocks, txes, outputs, key images

5 files changed, 294 insertions, 0 deletions

diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp
index 7648d657d..6560ce5c2 100644
--- a/src/blockchain_db/berkeleydb/db_bdb.cpp
+++ b/src/blockchain_db/berkeleydb/db_bdb.cpp
@@ -533,6 +533,126 @@ void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image)
         throw1(DB_ERROR("Error adding removal of key image to db transaction"));
 }
 
+bool BlockchainBDB::for_all_key_images(std::function<bool(const crypto::key_image&)> f) const
+{
+    LOG_PRINT_L3("BlockchainBDB::" << __func__);
+    check_open();
+
+    bdb_cur cur(DB_DEFAULT_TX, m_spent_keys);
+
+    Dbt_copy<crypto::key_image> k;
+    Dbt_copy<char> v;
+    bool ret = true;
+    int result;
+    while ((result = cur->get(&k, &v, DB_NEXT)) == 0)
+    {
+      if (!f(k))
+      {
+        ret = false;
+        break;
+      }
+    }
+    if (result != DB_NOTFOUND)
+      ret = false;
+
+    cur.close();
+    return ret;
+}
+
+bool BlockchainBDB::for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)> f) const
+{
+    LOG_PRINT_L3("BlockchainBDB::" << __func__);
+    check_open();
+
+    bdb_cur cur(DB_DEFAULT_TX, m_blocks);
+
+    Dbt_copy<uint64_t> k;
+    Dbt_safe v;
+    bool ret = true;
+    int result;
+    while ((result = cur->get(&k, &v, DB_NEXT)) == 0)
+    {
+      uint64_t height = k - 1;
+      blobdata bd;
+      bd.assign(reinterpret_cast<char*>(v.get_data()), v.get_size());
+      block b;
+      if (!parse_and_validate_block_from_blob(bd, b))
+          throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
+      crypto::hash hash;
+      if (!get_block_hash(b, hash))
+          throw0(DB_ERROR("Failed to get block hash from blob retrieved from the db"));
+      if (!f(height, hash, b))
+      {
+        ret = false;
+        break;
+      }
+    }
+    if (result != DB_NOTFOUND)
+      ret = false;
+
+    cur.close();
+    return ret;
+}
+
+bool BlockchainBDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f) const
+{
+    LOG_PRINT_L3("BlockchainBDB::" << __func__);
+    check_open();
+
+    bdb_cur cur(DB_DEFAULT_TX, m_txs);
+
+    Dbt_copy<crypto::hash> k;
+    Dbt_safe v;
+    bool ret = true;
+    int result;
+    while ((result = cur->get(&k, &v, DB_NEXT)) == 0)
+    {
+      blobdata bd;
+      bd.assign(reinterpret_cast<char*>(v.get_data()), v.get_size());
+      transaction tx;
+      if (!parse_and_validate_tx_from_blob(bd, tx))
+          throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
+      if (!f(k, tx))
+      {
+        ret = false;
+        break;
+      }
+    }
+    if (result != DB_NOTFOUND)
+      ret = false;
+
+    cur.close();
+    return ret;
+}
+
+bool BlockchainBDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const
+{
+    LOG_PRINT_L3("BlockchainBDB::" << __func__);
+    check_open();
+
+    bdb_cur cur(DB_DEFAULT_TX, m_output_amounts);
+
+    Dbt_copy<uint64_t> k;
+    Dbt_copy<uint32_t> v;
+    bool ret = true;
+    int result;
+    while ((result = cur->get(&k, &v, DB_NEXT)) == 0)
+    {
+      uint32_t global_index = v - 1;
+      tx_out_index toi = get_output_tx_and_index_from_global(global_index);
+      if (!f(k, toi.first, toi.second))
+      {
+        ret = false;
+        break;
+      }
+    }
+    if (result != DB_NOTFOUND)
+      ret = false;
+
+    cur.close();
+    return ret;
+}
+
 blobdata BlockchainBDB::output_to_blob(const tx_out& output) const
 {
     LOG_PRINT_L3("BlockchainBDB::" << __func__);
diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h
index 54edcf0ad..b51d745d8 100644
--- a/src/blockchain_db/berkeleydb/db_bdb.h
+++ b/src/blockchain_db/berkeleydb/db_bdb.h
@@ -375,6 +375,11 @@ private:
 
   void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &global_indices);
 
+  virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
+  virtual bool for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const;
+  virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const;
+  virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const;
+
   // Hard fork related storage
   virtual void set_hard_fork_starting_height(uint8_t version, uint64_t height);
   virtual uint64_t get_hard_fork_starting_height(uint8_t version) const;
diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h
index 24bf4024d..1666e57c5 100644
--- a/src/blockchain_db/blockchain_db.h
+++ b/src/blockchain_db/blockchain_db.h
@@ -493,6 +493,11 @@ public:
   // returns true if key image <img> is present in spent key images storage
   virtual bool has_key_image(const crypto::key_image& img) const = 0;
 
+  virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const = 0;
+  virtual bool for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const = 0;
+  virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const = 0;
+  virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const = 0;
+
   // Hard fork related storage
   virtual void set_hard_fork_starting_height(uint8_t version, uint64_t height) = 0;
   virtual uint64_t get_hard_fork_starting_height(uint8_t version) const = 0;
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index fb853a7f4..1e7078c67 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -1968,6 +1968,165 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
   return false;
 }
 
+bool BlockchainLMDB::for_all_key_images(std::function<bool(const crypto::key_image&)> f) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  mdb_txn_safe txn;
+  if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+    throw0(DB_ERROR("Failed to create a transaction for the db"));
+
+  MDB_val k;
+  MDB_val v;
+  bool ret = true;
+
+  lmdb_cur cur(txn, m_spent_keys);
+  MDB_cursor_op op = MDB_FIRST;
+  while (1)
+  {
+    int ret = mdb_cursor_get(cur, &k, &v, op);
+    op = MDB_NEXT;
+    if (ret == MDB_NOTFOUND)
+      break;
+    if (ret < 0)
+      throw0(DB_ERROR("Failed to enumerate key images"));
+    const crypto::key_image k_image = *(crypto::key_image*)k.mv_data;
+    if (!f(k_image)) {
+      ret = false;
+      break;
+    }
+  }
+
+  cur.close();
+  txn.commit();
+
+  return ret;
+}
+
+bool BlockchainLMDB::for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)> f) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  mdb_txn_safe txn;
+  if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+    throw0(DB_ERROR("Failed to create a transaction for the db"));
+
+  MDB_val k;
+  MDB_val v;
+  bool ret = true;
+
+  lmdb_cur cur(txn, m_blocks);
+  MDB_cursor_op op = MDB_FIRST;
+  while (1)
+  {
+    int ret = mdb_cursor_get(cur, &k, &v, op);
+    op = MDB_NEXT;
+    if (ret == MDB_NOTFOUND)
+      break;
+    if (ret)
+      throw0(DB_ERROR("Failed to enumerate blocks"));
+    uint64_t height = *(uint64_t*)k.mv_data;
+    blobdata bd;
+    bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+    block b;
+    if (!parse_and_validate_block_from_blob(bd, b))
+      throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
+    crypto::hash hash;
+    if (!get_block_hash(b, hash))
+        throw0(DB_ERROR("Failed to get block hash from blob retrieved from the db"));
+    if (!f(height, hash, b)) {
+      ret = false;
+      break;
+    }
+  }
+
+  cur.close();
+  txn.commit();
+
+  return ret;
+}
+
+bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  mdb_txn_safe txn;
+  if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+    throw0(DB_ERROR("Failed to create a transaction for the db"));
+
+  MDB_val k;
+  MDB_val v;
+  bool ret = true;
+
+  lmdb_cur cur(txn, m_txs);
+  MDB_cursor_op op = MDB_FIRST;
+  while (1)
+  {
+    int ret = mdb_cursor_get(cur, &k, &v, op);
+    op = MDB_NEXT;
+    if (ret == MDB_NOTFOUND)
+      break;
+    if (ret)
+      throw0(DB_ERROR("Failed to enumerate transactions"));
+    const crypto::hash hash = *(crypto::hash*)k.mv_data;
+    blobdata bd;
+    bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+    transaction tx;
+    if (!parse_and_validate_tx_from_blob(bd, tx))
+      throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
+    if (!f(hash, tx)) {
+      ret = false;
+      break;
+    }
+  }
+
+  cur.close();
+  txn.commit();
+
+  return ret;
+}
+
+bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  mdb_txn_safe txn;
+  if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+    throw0(DB_ERROR("Failed to create a transaction for the db"));
+
+  MDB_val k;
+  MDB_val v;
+  bool ret = true;
+
+  lmdb_cur cur(txn, m_output_amounts);
+  MDB_cursor_op op = MDB_FIRST;
+  while (1)
+  {
+    int ret = mdb_cursor_get(cur, &k, &v, op);
+    op = MDB_NEXT;
+    if (ret == MDB_NOTFOUND)
+      break;
+    if (ret)
+      throw0(DB_ERROR("Failed to enumerate outputs"));
+    uint64_t amount = *(uint64_t*)k.mv_data;
+    uint64_t global_index = *(uint64_t*)v.mv_data;
+    tx_out_index toi = get_output_tx_and_index_from_global(global_index);
+    if (!f(amount, toi.first, toi.second)) {
+      ret = false;
+      break;
+    }
+  }
+
+  cur.close();
+  txn.commit();
+
+  return ret;
+}
+
 // batch_num_blocks: (optional) Used to check if resize needed before batch transaction starts.
 void BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
 {
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index 380954295..6125ef2ef 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -189,6 +189,11 @@ public:
 
   virtual bool has_key_image(const crypto::key_image& img) const;
 
+  virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
+  virtual bool for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const;
+  virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const;
+  virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const;
+
   virtual uint64_t add_block( const block& blk
                             , const size_t& block_size
                             , const difficulty_type& cumulative_difficulty