wallet2, RPC: Optimize RPC calls for periodic refresh from 3 down to 1 call [release-v0.18]

2 files changed, 173 insertions, 60 deletions

diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp
index d9d851d47..e570ee109 100644
--- a/src/rpc/core_rpc_server.cpp
+++ b/src/rpc/core_rpc_server.cpp
@@ -598,88 +598,162 @@ namespace cryptonote
 
     CHECK_PAYMENT(req, res, 1);
 
-    // quick check for noop
-    if (!req.block_ids.empty())
-    {
-      uint64_t last_block_height;
-      crypto::hash last_block_hash;
-      m_core.get_blockchain_top(last_block_height, last_block_hash);
-      if (last_block_hash == req.block_ids.front())
-      {
-        res.start_height = 0;
-        res.current_height = m_core.get_current_blockchain_height();
-        res.status = CORE_RPC_STATUS_OK;
+    res.daemon_time = (uint64_t)time(NULL);
+    // Always set daemon time, and set it early rather than late, as delivering some incremental pool
+    // info twice because of slightly overlapping time intervals is no problem, whereas producing gaps
+    // and never delivering something is
+
+    bool get_blocks = false;
+    bool get_pool = false;
+    switch (req.requested_info)
+    {
+      case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY:
+        // Compatibility value 0: Clients that do not set 'requested_info' want blocks, and only blocks
+        get_blocks = true;
+        break;
+      case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL:
+        get_blocks = true;
+        get_pool = true;
+        break;
+      case COMMAND_RPC_GET_BLOCKS_FAST::POOL_ONLY:
+        get_pool = true;
+        break;
+      default:
+        res.status = "Failed, wrong requested info";
         return true;
-      }
     }
 
-    size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
-    if (m_rpc_payment)
+    res.pool_info_extent = COMMAND_RPC_GET_BLOCKS_FAST::NONE;
+
+    if (get_pool)
     {
-      max_blocks = res.credits / COST_PER_BLOCK;
-      if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
-        max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
-      if (max_blocks == 0)
+      const bool restricted = m_restricted && ctx;
+      const bool request_has_rpc_origin = ctx != NULL;
+      const bool allow_sensitive = !request_has_rpc_origin || !restricted;
+
+      bool incremental;
+      std::vector<tx_memory_pool::tx_details> added_pool_txs;
+      bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, added_pool_txs, res.removed_pool_txids, incremental);
+      if (success)
+      {
+        res.added_pool_txs.clear();
+        if (m_rpc_payment)
+        {
+          CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + res.removed_pool_txids.size() * COST_PER_POOL_HASH);
+        }
+        for (auto tx_detail: added_pool_txs)
+        {
+          COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info info;
+          info.tx_hash = cryptonote::get_transaction_hash(tx_detail.tx);
+          std::stringstream oss;
+          binary_archive<true> ar(oss);
+          bool r = ::serialization::serialize(ar, tx_detail.tx);
+          if (!r)
+          {
+            res.status = "Failed to serialize transaction";
+            return true;
+          }
+          info.tx_blob = oss.str();
+          info.double_spend_seen = tx_detail.double_spend_seen;
+          res.added_pool_txs.push_back(std::move(info));
+        }
+      }
+      if (success)
       {
-        res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
+        res.pool_info_extent = incremental ? COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL : COMMAND_RPC_GET_BLOCKS_FAST::FULL;
+      }
+      else
+      {
+        res.status = "Failed to get pool info";
         return true;
       }
     }
 
-    std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
-    if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
-    {
-      res.status = "Failed";
-      add_host_fail(ctx);
-      return true;
-    }
-
-    CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK);
-
-    size_t size = 0, ntxes = 0;
-    res.blocks.reserve(bs.size());
-    res.output_indices.reserve(bs.size());
-    for(auto& bd: bs)
+    if (get_blocks)
     {
-      res.blocks.resize(res.blocks.size()+1);
-      res.blocks.back().pruned = req.prune;
-      res.blocks.back().block = bd.first.first;
-      size += bd.first.first.size();
-      res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
-      ntxes += bd.second.size();
-      res.output_indices.back().indices.reserve(1 + bd.second.size());
-      if (req.no_miner_tx)
-        res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
-      res.blocks.back().txs.reserve(bd.second.size());
-      for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
+      // quick check for noop
+      if (!req.block_ids.empty())
       {
-        res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash});
-        i->second.clear();
-        i->second.shrink_to_fit();
-        size += res.blocks.back().txs.back().blob.size();
+        uint64_t last_block_height;
+        crypto::hash last_block_hash;
+        m_core.get_blockchain_top(last_block_height, last_block_hash);
+        if (last_block_hash == req.block_ids.front())
+        {
+          res.start_height = 0;
+          res.current_height = last_block_height + 1;
+          res.status = CORE_RPC_STATUS_OK;
+          return true;
+        }
       }
 
-      const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
-      if (n_txes_to_lookup > 0)
+      size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
+      if (m_rpc_payment)
       {
-        std::vector<std::vector<uint64_t>> indices;
-        bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices);
-        if (!r)
+        max_blocks = res.credits / COST_PER_BLOCK;
+        if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
+          max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
+        if (max_blocks == 0)
         {
-          res.status = "Failed";
+          res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
           return true;
         }
-        if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0))
+      }
+
+      std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
+      if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
+      {
+        res.status = "Failed";
+        add_host_fail(ctx);
+        return true;
+      }
+
+      CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK);
+
+      size_t size = 0, ntxes = 0;
+      res.blocks.reserve(bs.size());
+      res.output_indices.reserve(bs.size());
+      for(auto& bd: bs)
+      {
+        res.blocks.resize(res.blocks.size()+1);
+        res.blocks.back().pruned = req.prune;
+        res.blocks.back().block = bd.first.first;
+        size += bd.first.first.size();
+        res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
+        ntxes += bd.second.size();
+        res.output_indices.back().indices.reserve(1 + bd.second.size());
+        if (req.no_miner_tx)
+          res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
+        res.blocks.back().txs.reserve(bd.second.size());
+        for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
+        {
+          res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash});
+          i->second.clear();
+          i->second.shrink_to_fit();
+          size += res.blocks.back().txs.back().blob.size();
+        }
+
+        const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
+        if (n_txes_to_lookup > 0)
         {
-          res.status = "Failed";
-          return true;
+          std::vector<std::vector<uint64_t>> indices;
+          bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices);
+          if (!r)
+          {
+            res.status = "Failed";
+            return true;
+          }
+          if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0))
+          {
+            res.status = "Failed";
+            return true;
+          }
+          for (size_t i = 0; i < indices.size(); ++i)
+            res.output_indices.back().indices.push_back({std::move(indices[i])});
         }
-        for (size_t i = 0; i < indices.size(); ++i)
-          res.output_indices.back().indices.push_back({std::move(indices[i])});
       }
+      MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size);
     }
 
-    MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size);
     res.status = CORE_RPC_STATUS_OK;
     return true;
   }
diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h
index c1285d300..c52422e09 100644
--- a/src/rpc/core_rpc_server_commands_defs.h
+++ b/src/rpc/core_rpc_server_commands_defs.h
@@ -162,18 +162,29 @@ namespace cryptonote
   struct COMMAND_RPC_GET_BLOCKS_FAST
   {
 
+    enum REQUESTED_INFO
+    {
+      BLOCKS_ONLY = 0,
+      BLOCKS_AND_POOL = 1,
+      POOL_ONLY = 2
+    };
+
     struct request_t: public rpc_access_request_base
     {
+      uint8_t     requested_info;
       std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */
       uint64_t    start_height;
       bool        prune;
       bool        no_miner_tx;
+      uint64_t    pool_info_since;
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_request_base)
+        KV_SERIALIZE_OPT(requested_info, (uint8_t)0)
         KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids)
         KV_SERIALIZE(start_height)
         KV_SERIALIZE(prune)
         KV_SERIALIZE_OPT(no_miner_tx, false)
+        KV_SERIALIZE_OPT(pool_info_since, (uint64_t)0)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;
@@ -196,12 +207,36 @@ namespace cryptonote
       END_KV_SERIALIZE_MAP()
     };
 
+    struct pool_tx_info
+    {
+      crypto::hash tx_hash;
+      blobdata tx_blob;
+      bool double_spend_seen;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE_VAL_POD_AS_BLOB(tx_hash)
+        KV_SERIALIZE(tx_blob)
+        KV_SERIALIZE(double_spend_seen)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    enum POOL_INFO_EXTENT
+    {
+      NONE = 0,
+      INCREMENTAL = 1,
+      FULL = 2
+    };
+
     struct response_t: public rpc_access_response_base
     {
       std::vector<block_complete_entry> blocks;
       uint64_t    start_height;
       uint64_t    current_height;
       std::vector<block_output_indices> output_indices;
+      uint64_t    daemon_time;
+      uint8_t     pool_info_extent;
+      std::vector<pool_tx_info> added_pool_txs;
+      std::vector<crypto::hash> removed_pool_txids;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_response_base)
@@ -209,6 +244,10 @@ namespace cryptonote
         KV_SERIALIZE(start_height)
         KV_SERIALIZE(current_height)
         KV_SERIALIZE(output_indices)
+        KV_SERIALIZE(daemon_time)
+        KV_SERIALIZE(pool_info_extent)
+        KV_SERIALIZE(added_pool_txs)
+        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;