Move txpool to the database

Integration could go further (ie, return_tx_to_pool calls should
not be needed anymore, possibly other things).

poolstate.bin is now obsolete.

3 files changed, 321 insertions, 0 deletions

diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h
index f5710550b..9dd343f4b 100644
--- a/src/blockchain_db/blockchain_db.h
+++ b/src/blockchain_db/blockchain_db.h
@@ -124,6 +124,27 @@ struct tx_data_t
 };
 #pragma pack(pop)
 
+/**
+ * @brief a struct containing txpool per transaction metadata
+ */
+struct txpool_tx_meta_t
+{
+  crypto::hash max_used_block_id;
+  crypto::hash last_failed_id;
+  uint64_t blob_size;
+  uint64_t fee;
+  uint64_t max_used_block_height;
+  uint64_t last_failed_height;
+  uint64_t receive_time;
+  uint64_t last_relayed_time;
+  // 112 bytes
+  uint8_t kept_by_block;
+  uint8_t relayed;
+  uint8_t do_not_relay;
+
+  uint8_t padding[77]; // till 192 bytes
+};
+
 /***********************************
  * Exception Definitions
  ***********************************/
@@ -1252,6 +1273,71 @@ public:
   virtual bool has_key_image(const crypto::key_image& img) const = 0;
 
   /**
+   * @brief add a txpool transaction
+   *
+   * @param details the details of the transaction to add
+   */
+  virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) = 0;
+
+  /**
+   * @brief update a txpool transaction's metadata
+   *
+   * @param txid the txid of the transaction to update
+   * @param details the details of the transaction to update
+   */
+  virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& details) = 0;
+
+  /**
+   * @brief get the number of transactions in the txpool
+   */
+  virtual uint64_t get_txpool_tx_count() const = 0;
+
+  /**
+   * @brief check whether a txid is in the txpool
+   */
+  virtual bool txpool_has_tx(const crypto::hash &txid) const = 0;
+
+  /**
+   * @brief remove a txpool transaction
+   *
+   * @param txid the transaction id of the transation to remove
+   */
+  virtual void remove_txpool_tx(const crypto::hash& txid) = 0;
+
+  /**
+   * @brief get a txpool transaction's metadata
+   *
+   * @param txid the transaction id of the transation to lookup
+   *
+   * @return the metadata associated with that transaction
+   */
+  virtual txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const = 0;
+
+  /**
+   * @brief get a txpool transaction's blob
+   *
+   * @param txid the transaction id of the transation to lookup
+   *
+   * @return the blob for that transaction
+   */
+  virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0;
+
+  /**
+   * @brief runs a function over all txpool transactions
+   *
+   * The subclass should run the passed function for each txpool tx it has
+   * stored, passing the tx id and metadata as its parameters.
+   *
+   * If any call to the function returns false, the subclass should return
+   * false.  Otherwise, the subclass returns true.
+   *
+   * @param std::function fn the function to run
+   *
+   * @return false if the function returns false for any transaction, otherwise true
+   */
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false) const = 0;
+
+  /**
    * @brief runs a function over all key images stored
    *
    * The subclass should run the passed function for each key image it has
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index dab09e8f4..1f0831874 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -169,6 +169,9 @@ int compare_string(const MDB_val *a, const MDB_val *b)
  *
  * spent_keys       input hash   -
  *
+ * txpool_meta      txn hash     txn metadata
+ * txpool_blob      txn hash     txn blob
+ *
  * Note: where the data items are of uniform size, DUPFIXED tables have
  * been used to save space. In most of these cases, a dummy "zerokval"
  * key is used when accessing the table; the Key listed above will be
@@ -189,6 +192,9 @@ const char* const LMDB_OUTPUT_TXS = "output_txs";
 const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts";
 const char* const LMDB_SPENT_KEYS = "spent_keys";
 
+const char* const LMDB_TXPOOL_META = "txpool_meta";
+const char* const LMDB_TXPOOL_BLOB = "txpool_blob";
+
 const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights";
 const char* const LMDB_HF_VERSIONS = "hf_versions";
 
@@ -1164,6 +1170,9 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
 
   lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys");
 
+  lmdb_db_open(txn, LMDB_TXPOOL_META, MDB_CREATE, m_txpool_meta, "Failed to open db handle for m_txpool_meta");
+  lmdb_db_open(txn, LMDB_TXPOOL_BLOB, MDB_CREATE, m_txpool_blob, "Failed to open db handle for m_txpool_blob");
+
   // this subdb is dropped on sight, so it may not be present when we open the DB.
   // Since we use MDB_CREATE, we'll get an exception if we open read-only and it does not exist.
   // So we don't open for read-only, and also not drop below. It is not used elsewhere.
@@ -1181,6 +1190,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
   mdb_set_dupsort(txn, m_output_txs, compare_uint64);
   mdb_set_dupsort(txn, m_block_info, compare_uint64);
 
+  mdb_set_compare(txn, m_txpool_meta, compare_hash32);
+  mdb_set_compare(txn, m_txpool_blob, compare_hash32);
   mdb_set_compare(txn, m_properties, compare_string);
 
   if (!(mdb_flags & MDB_RDONLY))
@@ -1429,6 +1440,211 @@ void BlockchainLMDB::unlock()
       auto_txn.commit(); \
   } while(0)
 
+void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t &meta)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  mdb_txn_cursors *m_cursors = &m_wcursors;
+
+  CURSOR(txpool_meta)
+  CURSOR(txpool_blob)
+
+  const crypto::hash txid = get_transaction_hash(tx);
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  MDB_val v = {sizeof(meta), (void *)&meta};
+  if (auto result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) {
+    if (result == MDB_KEYEXIST)
+      throw1(DB_ERROR("Attempting to add txpool tx metadata that's already in the db"));
+    else
+      throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str()));
+  }
+  MDB_val_copy<cryptonote::blobdata> blob_val(tx_to_blob(tx));
+  if (auto result = mdb_cursor_put(m_cur_txpool_blob, &k, &blob_val, MDB_NODUPDATA)) {
+    if (result == MDB_KEYEXIST)
+      throw1(DB_ERROR("Attempting to add txpool tx blob that's already in the db"));
+    else
+      throw1(DB_ERROR(lmdb_error("Error adding txpool tx blob to db transaction: ", result).c_str()));
+  }
+}
+
+void BlockchainLMDB::update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  mdb_txn_cursors *m_cursors = &m_wcursors;
+
+  CURSOR(txpool_meta)
+  CURSOR(txpool_blob)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  MDB_val v;
+  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET);
+  if (result != 0)
+    throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta to update: ", result).c_str()));
+  result = mdb_cursor_del(m_cur_txpool_meta, 0);
+  if (result)
+    throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx metadata to db transaction: ", result).c_str()));
+  v = MDB_val({sizeof(meta), (void *)&meta});
+  if ((result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) != 0) {
+    if (result == MDB_KEYEXIST)
+      throw1(DB_ERROR("Attempting to add txpool tx metadata that's already in the db"));
+    else
+      throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str()));
+  }
+}
+
+uint64_t BlockchainLMDB::get_txpool_tx_count() const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  int result;
+
+  MDB_stat db_stats;
+  if ((result = mdb_stat(m_txn, m_txpool_meta, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_txpool_meta: ", result).c_str()));
+
+  TXN_POSTFIX_RDONLY();
+
+  return db_stats.ms_entries;
+}
+
+bool BlockchainLMDB::txpool_has_tx(const crypto::hash& txid) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(txpool_meta)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET);
+  if (result != 0 && result != MDB_NOTFOUND)
+    throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));
+  TXN_POSTFIX_RDONLY();
+  return result != MDB_NOTFOUND;
+}
+
+void BlockchainLMDB::remove_txpool_tx(const crypto::hash& txid)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  mdb_txn_cursors *m_cursors = &m_wcursors;
+
+  CURSOR(txpool_meta)
+  CURSOR(txpool_blob)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET);
+  if (result != 0 && result != MDB_NOTFOUND)
+    throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta to remove: ", result).c_str()));
+  if (!result)
+  {
+    result = mdb_cursor_del(m_cur_txpool_meta, 0);
+    if (result)
+      throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx metadata to db transaction: ", result).c_str()));
+  }
+  result = mdb_cursor_get(m_cur_txpool_blob, &k, NULL, MDB_SET);
+  if (result != 0 && result != MDB_NOTFOUND)
+    throw1(DB_ERROR(lmdb_error("Error finding txpool tx blob to remove: ", result).c_str()));
+  if (!result)
+  {
+    result = mdb_cursor_del(m_cur_txpool_blob, 0);
+    if (result)
+      throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx blob to db transaction: ", result).c_str()));
+  }
+}
+
+txpool_tx_meta_t BlockchainLMDB::get_txpool_tx_meta(const crypto::hash& txid) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(txpool_meta)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  MDB_val v;
+  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET);
+  if (result != 0)
+      throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));
+
+  const txpool_tx_meta_t meta = *(const txpool_tx_meta_t*)v.mv_data;
+  TXN_POSTFIX_RDONLY();
+  return meta;
+}
+
+cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(txpool_blob)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  MDB_val v;
+  auto result = mdb_cursor_get(m_cur_txpool_blob, &k, &v, MDB_SET);
+  if (result != 0)
+      throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));
+
+  blobdata bd;
+  bd.assign(reinterpret_cast<const char*>(v.mv_data), v.mv_size);
+  TXN_POSTFIX_RDONLY();
+  return bd;
+}
+
+bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(txpool_meta);
+  RCURSOR(txpool_blob);
+
+  MDB_val k;
+  MDB_val v;
+  bool ret = true;
+
+  MDB_cursor_op op = MDB_FIRST;
+  while (1)
+  {
+    int result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, op);
+    op = MDB_NEXT;
+    if (result == MDB_NOTFOUND)
+      break;
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str()));
+    const crypto::hash txid = *(const crypto::hash*)k.mv_data;
+    const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data;
+    const cryptonote::blobdata *passed_bd = NULL;
+    cryptonote::blobdata bd;
+    if (include_blob)
+    {
+      MDB_val b;
+      result = mdb_cursor_get(m_cur_txpool_blob, &k, &b, MDB_SET);
+      if (result == MDB_NOTFOUND)
+        throw0(DB_ERROR("Failed to find txpool tx blob to match metadata"));
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx blob: ", result).c_str()));
+      bd.assign(reinterpret_cast<const char*>(b.mv_data), b.mv_size);
+      passed_bd = &bd;
+    }
+
+    if (!f(txid, meta, passed_bd)) {
+      ret = false;
+      break;
+    }
+  }
+
+  TXN_POSTFIX_RDONLY();
+
+  return ret;
+}
+
 bool BlockchainLMDB::block_exists(const crypto::hash& h, uint64_t *height) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index 8a5677566..02f57ce18 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -55,6 +55,9 @@ typedef struct mdb_txn_cursors
 
   MDB_cursor *m_txc_spent_keys;
 
+  MDB_cursor *m_txc_txpool_meta;
+  MDB_cursor *m_txc_txpool_blob;
+
   MDB_cursor *m_txc_hf_versions;
 } mdb_txn_cursors;
 
@@ -67,6 +70,8 @@ typedef struct mdb_txn_cursors
 #define m_cur_tx_indices	m_cursors->m_txc_tx_indices
 #define m_cur_tx_outputs	m_cursors->m_txc_tx_outputs
 #define m_cur_spent_keys	m_cursors->m_txc_spent_keys
+#define m_cur_txpool_meta	m_cursors->m_txc_txpool_meta
+#define m_cur_txpool_blob	m_cursors->m_txc_txpool_blob
 #define m_cur_hf_versions	m_cursors->m_txc_hf_versions
 
 typedef struct mdb_rflags
@@ -81,6 +86,8 @@ typedef struct mdb_rflags
   bool m_rf_tx_indices;
   bool m_rf_tx_outputs;
   bool m_rf_spent_keys;
+  bool m_rf_txpool_meta;
+  bool m_rf_txpool_blob;
   bool m_rf_hf_versions;
 } mdb_rflags;
 
@@ -232,6 +239,15 @@ public:
 
   virtual bool has_key_image(const crypto::key_image& img) const;
 
+  virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& meta);
+  virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta);
+  virtual uint64_t get_txpool_tx_count() const;
+  virtual bool txpool_has_tx(const crypto::hash &txid) const;
+  virtual void remove_txpool_tx(const crypto::hash& txid);
+  virtual txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const;
+  virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const;
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false) const;
+
   virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
   virtual bool for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const;
   virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const;
@@ -364,6 +380,9 @@ private:
 
   MDB_dbi m_spent_keys;
 
+  MDB_dbi m_txpool_meta;
+  MDB_dbi m_txpool_blob;
+
   MDB_dbi m_hf_starting_heights;
   MDB_dbi m_hf_versions;