Merge pull request #764

f746c9d minor corrections/clarifications (Thomas Winget)
c6bb201 Transaction pool documentation (and some cleanup) (Thomas Winget)

2 files changed, 374 insertions, 74 deletions

diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp
index 5d67acdd2..49fa4aa57 100644
--- a/src/cryptonote_core/tx_pool.cpp
+++ b/src/cryptonote_core/tx_pool.cpp
@@ -54,6 +54,11 @@ namespace cryptonote
 {
   namespace
   {
+    //TODO: constants such as these should at least be in the header,
+    //      but probably somewhere more accessible to the rest of the
+    //      codebase.  As it stands, it is at best nontrivial to test
+    //      whether or not changing these parameters (or adding new)
+    //      will work correctly.
     size_t const TRANSACTION_SIZE_LIMIT_V1 = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE);
     size_t const TRANSACTION_SIZE_LIMIT_V2 = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE);
     time_t const MIN_RELAY_TIME = (60 * 5); // only start re-relaying transactions after that many seconds
@@ -116,11 +121,12 @@ namespace cryptonote
       return false;
     }
 
+    // fee per kilobyte, size rounded up.
     uint64_t fee = inputs_amount - outputs_amount;
     uint64_t needed_fee = blob_size / 1024;
     needed_fee += (blob_size % 1024) ? 1 : 0;
     needed_fee *= FEE_PER_KB;
-    if (!kept_by_block && fee < needed_fee /*&& fee < MINING_ALLOWED_LEGACY_FEE*/)
+    if (!kept_by_block && fee < needed_fee)
     {
       LOG_PRINT_L1("transaction fee is not enough: " << print_money(fee) << ", minimum fee: " << print_money(needed_fee));
       tvc.m_verifivation_failed = true;
@@ -135,7 +141,9 @@ namespace cryptonote
       return false;
     }
 
-    //check key images for transaction if it is not kept by block
+    // if the transaction came from a block popped from the chain,
+    // don't check if we have its key images as spent.
+    // TODO: Investigate why not?
     if(!kept_by_block)
     {
       if(have_tx_keyimges_as_spent(tx))
@@ -163,9 +171,10 @@ namespace cryptonote
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     if(!ch_inp_res)
     {
+      // if the transaction was valid before (kept_by_block), then it
+      // may become valid again, so ignore the failed inputs check.
       if(kept_by_block)
       {
-        //anyway add this transaction to pool, because it related to block
         auto txd_p = m_transactions.insert(transactions_container::value_type(id, tx_details()));
         CHECK_AND_ASSERT_MES(txd_p.second, false, "transaction already exists at inserting in memory pool");
         txd_p.first->second.blob_size = blob_size;
@@ -207,13 +216,14 @@ namespace cryptonote
         tvc.m_should_be_relayed = true;
     }
 
+    // assume failure during verification steps until success is certain
     tvc.m_verifivation_failed = true;
-    //update image_keys container, here should everything goes ok.
+
     BOOST_FOREACH(const auto& in, tx.vin)
     {
       CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, txin, false);
       std::unordered_set<crypto::hash>& kei_image_set = m_spent_key_images[txin.k_image];
-      CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: keeped_by_block=" << kept_by_block
+      CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: kept_by_block=" << kept_by_block
                                           << ",  kei_image_set.size()=" << kei_image_set.size() << ENDL << "txin.k_image=" << txin.k_image << ENDL
                                           << "tx_id=" << id );
       auto ins_res = kei_image_set.insert(id);
@@ -223,7 +233,7 @@ namespace cryptonote
     tvc.m_verifivation_failed = false;
 
     m_txs_by_fee.emplace((double)blob_size / fee, id);
-    //succeed
+
     return true;
   }
   //---------------------------------------------------------------------------------
@@ -235,6 +245,9 @@ namespace cryptonote
     return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, version);
   }
   //---------------------------------------------------------------------------------
+  //FIXME: Can return early before removal of all of the key images.
+  //       At the least, need to make sure that a false return here
+  //       is treated properly.  Should probably not return early, however.
   bool tx_memory_pool::remove_transaction_keyimages(const transaction& tx)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -301,7 +314,7 @@ namespace cryptonote
     );
   }
   //---------------------------------------------------------------------------------
-  //proper tx_pool handling courtesy of CryptoZoidberg and Boolberry
+  //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::remove_stuck_transactions()
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -332,6 +345,7 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------------------------
+  //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::transaction>> &txs) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -380,6 +394,7 @@ namespace cryptonote
       txs.push_back(tx_vt.second.tx);
   }
   //------------------------------------------------------------------
+  //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -556,6 +571,7 @@ namespace cryptonote
     return ss.str();
   }
   //---------------------------------------------------------------------------------
+  //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee)
   {
     // Warning: This function takes already_generated_
@@ -646,6 +662,7 @@ namespace cryptonote
     return n_removed;
   }
   //---------------------------------------------------------------------------------
+  //TODO: investigate whether only ever returning true is correct
   bool tx_memory_pool::init(const std::string& config_folder)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -679,6 +696,7 @@ namespace cryptonote
   }
 
   //---------------------------------------------------------------------------------
+  //TODO: investigate whether only ever returning true is correct
   bool tx_memory_pool::deinit()
   {
     if (m_config_folder.empty())
diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h
index 71febcab6..84e11eeff 100644
--- a/src/cryptonote_core/tx_pool.h
+++ b/src/cryptonote_core/tx_pool.h
@@ -57,7 +57,9 @@ namespace cryptonote
   /*                                                                      */
   /************************************************************************/
 
+  //! pair of <transaction fee, transaction hash> for organization
   typedef std::pair<double, crypto::hash> tx_by_fee_entry;
+
   class txCompare
   {
   public:
@@ -71,47 +73,255 @@ namespace cryptonote
     }
   };
 
+  //! container for sorting transactions by fee per unit size
   typedef std::set<tx_by_fee_entry, txCompare> sorted_tx_container;
 
+  /**
+   * @brief Transaction pool, handles transactions which are not part of a block
+   *
+   * This class handles all transactions which have been received, but not as
+   * part of a block.
+   *
+   * This handling includes:
+   *   storing the transactions
+   *   organizing the transactions by fee per size
+   *   taking/giving transactions to and from various other components
+   *   saving the transactions to disk on shutdown
+   *   helping create a new block template by choosing transactions for it
+   *
+   */
   class tx_memory_pool: boost::noncopyable
   {
   public:
 #if BLOCKCHAIN_DB == DB_LMDB
+    /**
+     * @brief Constructor
+     *
+     * @param bchs a Blockchain class instance, for getting chain info
+     */
     tx_memory_pool(Blockchain& bchs);
 #else
     tx_memory_pool(blockchain_storage& bchs);
 #endif
-    bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, uint8_t version);
-    bool add_tx(const transaction &tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, uint8_t version);
-    //gets tx and remove it from pool
+
+
+    /**
+     * @copydoc add_tx(const transaction&, tx_verification_context&, bool, bool, uint8_t)
+     *
+     * @param id the transaction's hash
+     * @param blob_size the transaction's size
+     */
+    bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, uint8_t version);
+
+    /**
+     * @brief add a transaction to the transaction pool
+     *
+     * Most likely the transaction will come from the network, but it is
+     * also possible for transactions to come from popped blocks during
+     * a reorg, or from local clients creating a transaction and
+     * submitting it to the network
+     *
+     * @param tx the transaction to be added
+     * @param tvc return-by-reference status about the transaction verification
+     * @param kept_by_block has this transaction been in a block?
+     * @param relayed was this transaction from the network or a local client?
+     * @param version the version used to create the transaction
+     *
+     * @return true if the transaction passes validations, otherwise false
+     */
+    bool add_tx(const transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool relayed, uint8_t version);
+
+    /**
+     * @brief takes a transaction with the given hash from the pool
+     *
+     * @param id the hash of the transaction
+     * @param tx return-by-reference the transaction taken
+     * @param blob_size return-by-reference the transaction's size
+     * @param fee the transaction fee
+     * @param relayed return-by-reference was transaction relayed to us by the network?
+     *
+     * @return true unless the transaction cannot be found in the pool
+     */
     bool take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed);
 
+    /**
+     * @brief checks if the pool has a transaction with the given hash
+     *
+     * @param id the hash to look for
+     *
+     * @return true if the transaction is in the pool, otherwise false
+     */
     bool have_tx(const crypto::hash &id) const;
+
+    /**
+     * @brief action to take when notified of a block added to the blockchain
+     *
+     * Currently does nothing
+     *
+     * @param new_block_height the height of the blockchain after the change
+     * @param top_block_id the hash of the new top block
+     *
+     * @return true
+     */
     bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id);
+
+    /**
+     * @brief action to take when notified of a block removed from the blockchain
+     *
+     * Currently does nothing
+     *
+     * @param new_block_height the height of the blockchain after the change
+     * @param top_block_id the hash of the new top block
+     *
+     * @return true
+     */
     bool on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id);
+
+    /**
+     * @brief action to take periodically
+     *
+     * Currently checks transaction pool for stale ("stuck") transactions
+     */
     void on_idle();
 
+    /**
+     * @brief locks the transaction pool
+     */
     void lock() const;
+
+    /**
+     * @brief unlocks the transaction pool
+     */
     void unlock() const;
 
     // load/store operations
+
+    /**
+     * @brief loads pool state (if any) from disk, and initializes pool
+     *
+     * @param config_folder folder name where pool state will be
+     *
+     * @return true
+     */
     bool init(const std::string& config_folder);
+
+    /**
+     * @brief attempts to save the transaction pool state to disk
+     *
+     * Currently fails (returns false) if the data directory from init()
+     * does not exist and cannot be created, but returns true even if
+     * saving to disk is unsuccessful.
+     *
+     * @return true in most cases (see above)
+     */
     bool deinit();
+
+    /**
+     * @brief Chooses transactions for a block to include
+     *
+     * @param bl return-by-reference the block to fill in with transactions
+     * @param median_size the current median block size
+     * @param already_generated_coins the current total number of coins "minted"
+     * @param total_size return-by-reference the total size of the new block
+     * @param fee return-by-reference the total of fees from the included transactions
+     *
+     * @return true
+     */
     bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee);
+
+    /**
+     * @brief get a list of all transactions in the pool
+     *
+     * @param txs return-by-reference the list of transactions
+     */
     void get_transactions(std::list<transaction>& txs) const;
+
+    /**
+     * @brief get information about all transactions and key images in the pool
+     *
+     * see documentation on tx_info and spent_key_image_info for more details
+     *
+     * @param tx_infos return-by-reference the transactions' information
+     * @param key_image_infos return-by-reference the spent key images' information
+     *
+     * @return true
+     */
     bool get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const;
+
+    /**
+     * @brief get a specific transaction from the pool
+     *
+     * @param h the hash of the transaction to get
+     * @param tx return-by-reference the transaction requested
+     *
+     * @return true if the transaction is found, otherwise false
+     */
     bool get_transaction(const crypto::hash& h, transaction& tx) const;
+
+    /**
+     * @brief get a list of all relayable transactions and their hashes
+     *
+     * "relayable" in this case means:
+     *   nonzero fee
+     *   hasn't been relayed too recently
+     *   isn't old enough that relaying it is considered harmful
+     *
+     * @param txs return-by-reference the transactions and their hashes
+     *
+     * @return true
+     */
     bool get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::transaction>>& txs) const;
+
+    /**
+     * @brief tell the pool that certain transactions were just relayed
+     *
+     * @param txs the list of transactions (and their hashes)
+     */
     void set_relayed(const std::list<std::pair<crypto::hash, cryptonote::transaction>>& txs);
+
+    /**
+     * @brief get the total number of transactions in the pool
+     *
+     * @return the number of transactions in the pool
+     */
     size_t get_transactions_count() const;
+
+    /**
+     * @brief get a string containing human-readable pool information
+     *
+     * @param short_format whether to use a shortened format for the info
+     *
+     * @return the string
+     */
     std::string print_pool(bool short_format) const;
+
+    /**
+     * @brief remove transactions from the pool which are no longer valid
+     *
+     * With new versions of the currency, what conditions render a transaction
+     * invalid may change.  This function clears those which were received
+     * before a version change and no longer conform to requirements.
+     *
+     * @param version the version the transactions must conform to
+     *
+     * @return the number of transactions removed
+     */
     size_t validate(uint8_t version);
 
-    /*bool flush_pool(const std::strig& folder);
-    bool inflate_pool(const std::strig& folder);*/
 
 #define CURRENT_MEMPOOL_ARCHIVE_VER    11
 
+    /**
+     * @brief serialize the transaction pool to/from disk
+     *
+     * If the archive version passed is older than the version compiled
+     * in, this function does nothing, as it cannot deserialize after a
+     * format change.
+     *
+     * @tparam archive_t the archive class
+     * @param a the archive to serialize to/from
+     * @param version the archive version
+     */
     template<class archive_t>
     void serialize(archive_t & a, const unsigned int version)
     {
@@ -123,97 +333,169 @@ namespace cryptonote
       a & m_timed_out_transactions;
     }
 
+    /**
+     * @brief information about a single transaction
+     */
     struct tx_details
     {
-      transaction tx;
-      size_t blob_size;
-      uint64_t fee;
-      crypto::hash max_used_block_id;
-      uint64_t max_used_block_height;
-      bool kept_by_block;
-      //
-      uint64_t last_failed_height;
+      transaction tx;  //!< the transaction
+      size_t blob_size;  //!< the transaction's size
+      uint64_t fee;  //!< the transaction's fee amount
+      crypto::hash max_used_block_id;  //!< the hash of the highest block referenced by an input
+      uint64_t max_used_block_height;  //!< the height of the highest block referenced by an input
+
+      //! whether or not the transaction has been in a block before
+      /*! if the transaction was returned to the pool from the blockchain
+       *  due to a reorg, then this will be true
+       */
+      bool kept_by_block;  
+
+      //! the highest block the transaction referenced when last checking it failed
+      /*! if verifying a transaction's inputs fails, it's possible this is due
+       *  to a reorg since it was created (if it used recently created outputs
+       *  as inputs).
+       */
+      uint64_t last_failed_height;  
+
+      //! the hash of the highest block the transaction referenced when last checking it failed
+      /*! if verifying a transaction's inputs fails, it's possible this is due
+       *  to a reorg since it was created (if it used recently created outputs
+       *  as inputs).
+       */
       crypto::hash last_failed_id;
-      time_t receive_time;
 
-      time_t last_relayed_time;
-      bool relayed;
+      time_t receive_time;  //!< the time when the transaction entered the pool
+
+      time_t last_relayed_time;  //!< the last time the transaction was relayed to the network
+      bool relayed;  //!< whether or not the transaction has been relayed to the network
     };
 
   private:
+
+    /**
+     * @brief remove old transactions from the pool
+     *
+     * After a certain time, it is assumed that a transaction which has not
+     * yet been mined will likely not be mined.  These transactions are removed
+     * from the pool to avoid buildup.
+     *
+     * @return true
+     */
     bool remove_stuck_transactions();
+
+    /**
+     * @brief check if a transaction in the pool has a given spent key image
+     *
+     * @param key_im the spent key image to look for
+     *
+     * @return true if the spent key image is present, otherwise false
+     */
     bool have_tx_keyimg_as_spent(const crypto::key_image& key_im) const;
+
+    /**
+     * @brief check if any spent key image in a transaction is in the pool
+     *
+     * Checks if any of the spent key images in a given transaction are present
+     * in any of the transactions in the transaction pool.
+     *
+     * @note see tx_pool::have_tx_keyimg_as_spent
+     *
+     * @param tx the transaction to check spent key images of
+     *
+     * @return true if any spent key images are present in the pool, otherwise false
+     */
     bool have_tx_keyimges_as_spent(const transaction& tx) const;
+
+    /**
+     * @brief forget a transaction's spent key images
+     *
+     * Spent key images are stored separately from transactions for
+     * convenience/speed, so this is part of the process of removing
+     * a transaction from the pool.
+     *
+     * @param tx the transaction
+     *
+     * @return false if any key images to be removed cannot be found, otherwise true
+     */
     bool remove_transaction_keyimages(const transaction& tx);
+
+    /**
+     * @brief check if any of a transaction's spent key images are present in a given set
+     *
+     * @param kic the set of key images to check against
+     * @param tx the transaction to check
+     *
+     * @return true if any key images present in the set, otherwise false
+     */
     static bool have_key_images(const std::unordered_set<crypto::key_image>& kic, const transaction& tx);
+
+    /**
+     * @brief append the key images from a transaction to the given set
+     *
+     * @param kic the set of key images to append to
+     * @param tx the transaction
+     *
+     * @return false if any append fails, otherwise true
+     */
     static bool append_key_images(std::unordered_set<crypto::key_image>& kic, const transaction& tx);
 
+    /**
+     * @brief check if a transaction is a valid candidate for inclusion in a block
+     *
+     * @param txd the transaction to check (and info about it)
+     *
+     * @return true if the transaction is good to go, otherwise false
+     */
     bool is_transaction_ready_to_go(tx_details& txd) const;
+
+    //! map transactions (and related info) by their hashes
     typedef std::unordered_map<crypto::hash, tx_details > transactions_container;
+
+    //TODO: confirm the below comments and investigate whether or not this
+    //      is the desired behavior
+    //! map key images to transactions which spent them
+    /*! this seems odd, but it seems that multiple transactions can exist
+     *  in the pool which both have the same spent key.  This would happen
+     *  in the event of a reorg where someone creates a new/different
+     *  transaction on the assumption that the original will not be in a
+     *  block again.
+     */
     typedef std::unordered_map<crypto::key_image, std::unordered_set<crypto::hash> > key_images_container;
 
-    mutable epee::critical_section m_transactions_lock;
-    transactions_container m_transactions;
-    key_images_container m_spent_key_images;
+    mutable epee::critical_section m_transactions_lock;  //!< lock for the pool
+    transactions_container m_transactions;  //!< container for transactions in the pool
+
+    //! container for spent key images from the transactions in the pool
+    key_images_container m_spent_key_images;  
+
+    //TODO: this time should be a named constant somewhere, not hard-coded
+    //! interval on which to check for stale/"stuck" transactions
     epee::math_helper::once_a_time_seconds<30> m_remove_stuck_tx_interval;
 
-    //TODO: add fee_per_kb element to type tx_details and replace this
-    //functionality by just making m_transactions a std::set
-    sorted_tx_container m_txs_by_fee;
+    //TODO: look into doing this better
+    sorted_tx_container m_txs_by_fee;  //!< container for transactions organized by fee per size
 
+    /**
+     * @brief get an iterator to a transaction in the sorted container
+     *
+     * @param id the hash of the transaction to look for
+     *
+     * @return an iterator, possibly to the end of the container if not found
+     */
     sorted_tx_container::iterator find_tx_in_sorted_container(const crypto::hash& id) const;
 
+    //! transactions which are unlikely to be included in blocks
+    /*! These transactions are kept in RAM in case they *are* included
+     *  in a block eventually, but this container is not saved to disk.
+     */
     std::unordered_set<crypto::hash> m_timed_out_transactions;
 
-    //transactions_container m_alternative_transactions;
-
-    std::string m_config_folder;
+    std::string m_config_folder;  //!< the folder to save state to
 #if BLOCKCHAIN_DB == DB_LMDB
-    Blockchain& m_blockchain;
+    Blockchain& m_blockchain;  //!< reference to the Blockchain object
 #else
     blockchain_storage& m_blockchain;
 #endif
-    /************************************************************************/
-    /*                                                                      */
-    /************************************************************************/
-    /*class inputs_visitor: public boost::static_visitor<bool>
-    {
-      key_images_container& m_spent_keys;
-    public:
-      inputs_visitor(key_images_container& spent_keys): m_spent_keys(spent_keys)
-      {}
-      bool operator()(const txin_to_key& tx) const
-      {
-        auto pr = m_spent_keys.insert(tx.k_image);
-        CHECK_AND_ASSERT_MES(pr.second, false, "Tried to insert transaction with input seems already spent, input: " << epee::string_tools::pod_to_hex(tx.k_image));
-        return true;
-      }
-      bool operator()(const txin_gen& tx) const
-      {
-        CHECK_AND_ASSERT_MES(false, false, "coinbase transaction in memory pool");
-        return false;
-      }
-      bool operator()(const txin_to_script& tx) const {return false;}
-      bool operator()(const txin_to_scripthash& tx) const {return false;}
-    }; */
-    /************************************************************************/
-    /*                                                                      */
-    /************************************************************************/
-    class amount_visitor: public boost::static_visitor<uint64_t>
-    {
-    public:
-      uint64_t operator()(const txin_to_key& tx) const
-      {
-        return tx.amount;
-      }
-      uint64_t operator()(const txin_gen& tx) const
-      {
-        CHECK_AND_ASSERT_MES(false, false, "coinbase transaction in memory pool");
-        return 0;
-      }
-      uint64_t operator()(const txin_to_script& tx) const {return 0;}
-      uint64_t operator()(const txin_to_scripthash& tx) const {return 0;}
-    };
 
 #if BLOCKCHAIN_DB == DB_LMDB
 #else