Merge pull request #4687

6097472a Update ZMQ fee estimate and add ZMQ output distribution (Lee Clagett)

10 files changed, 226 insertions, 80 deletions

diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt
index 4f8f96524..8b4c27e3e 100644
--- a/src/rpc/CMakeLists.txt
+++ b/src/rpc/CMakeLists.txt
@@ -27,7 +27,8 @@
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 set(rpc_base_sources
-  rpc_args.cpp)
+  rpc_args.cpp
+  rpc_handler.cpp)
 
 set(rpc_sources
   core_rpc_server.cpp
@@ -43,7 +44,8 @@ set(daemon_rpc_server_sources
 
 
 set(rpc_base_headers
-  rpc_args.h)
+  rpc_args.h
+  rpc_handler.h)
 
 set(rpc_headers)
 
@@ -63,7 +65,6 @@ set(daemon_rpc_server_private_headers
   message.h
   daemon_messages.h
   daemon_handler.h
-  rpc_handler.h
   zmq_server.h)
 
 
diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp
index aa9d3d64b..574f6c126 100644
--- a/src/rpc/core_rpc_server.cpp
+++ b/src/rpc/core_rpc_server.cpp
@@ -45,6 +45,7 @@ using namespace epee;
 #include "storages/http_abstract_invoke.h"
 #include "crypto/hash.h"
 #include "rpc/rpc_args.h"
+#include "rpc/rpc_handler.h"
 #include "core_rpc_server_error_codes.h"
 #include "p2p/net_node.h"
 #include "version.h"
@@ -2117,62 +2118,15 @@ namespace cryptonote
       const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1);
       for (uint64_t amount: req.amounts)
       {
-        static struct D
-        {
-          boost::mutex mutex;
-          std::vector<uint64_t> cached_distribution;
-          uint64_t cached_from, cached_to, cached_start_height, cached_base;
-          bool cached;
-          D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {}
-        } d;
-        boost::unique_lock<boost::mutex> lock(d.mutex);
-
-        if (d.cached && amount == 0 && d.cached_from == req.from_height && d.cached_to == req_to_height)
-        {
-          res.distributions.push_back({amount, d.cached_start_height, req.binary, d.cached_distribution, d.cached_base});
-          if (!req.cumulative)
-          {
-            auto &distribution = res.distributions.back().distribution;
-            for (size_t n = distribution.size() - 1; n > 0; --n)
-              distribution[n] -= distribution[n-1];
-            distribution[0] -= d.cached_base;
-          }
-          continue;
-        }
-
-        std::vector<uint64_t> distribution;
-        uint64_t start_height, base;
-        if (!m_core.get_output_distribution(amount, req.from_height, req_to_height, start_height, distribution, base))
+        auto data = rpc::RpcHandler::get_output_distribution(m_core, amount, req.from_height, req_to_height, req.cumulative);
+        if (!data)
         {
           error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
-          error_resp.message = "Failed to get rct distribution";
+          error_resp.message = "Failed to get output distribution";
           return false;
         }
-        if (req_to_height > 0 && req_to_height >= req.from_height)
-        {
-          uint64_t offset = std::max(req.from_height, start_height);
-          if (offset <= req_to_height && req_to_height - offset + 1 < distribution.size())
-            distribution.resize(req_to_height - offset + 1);
-        }
-
-        if (amount == 0)
-        {
-          d.cached_from = req.from_height;
-          d.cached_to = req_to_height;
-          d.cached_distribution = distribution;
-          d.cached_start_height = start_height;
-          d.cached_base = base;
-          d.cached = true;
-        }
-
-        if (!req.cumulative)
-        {
-          for (size_t n = distribution.size() - 1; n > 0; --n)
-            distribution[n] -= distribution[n-1];
-          distribution[0] -= base;
-        }
 
-        res.distributions.push_back({amount, start_height, req.binary, std::move(distribution), base});
+        res.distributions.push_back({std::move(*data), amount, req.binary});
       }
     }
     catch (const std::exception &e)
diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h
index b229841d6..8e8df7a52 100644
--- a/src/rpc/core_rpc_server_commands_defs.h
+++ b/src/rpc/core_rpc_server_commands_defs.h
@@ -33,6 +33,7 @@
 #include "cryptonote_basic/cryptonote_basic.h"
 #include "cryptonote_basic/difficulty.h"
 #include "crypto/hash.h"
+#include "rpc/rpc_handler.h"
 
 namespace cryptonote
 {
@@ -2233,21 +2234,19 @@ namespace cryptonote
 
     struct distribution
     {
+      rpc::output_distribution_data data;
       uint64_t amount;
-      uint64_t start_height;
       bool binary;
-      std::vector<uint64_t> distribution;
-      uint64_t base;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(amount)
-        KV_SERIALIZE(start_height)
+        KV_SERIALIZE_N(data.start_height, "start_height")
         KV_SERIALIZE(binary)
         if (this_ref.binary)
-          KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution)
+          KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution")
         else
-          KV_SERIALIZE(distribution)
-        KV_SERIALIZE(base)
+          KV_SERIALIZE_N(data.distribution, "distribution")
+        KV_SERIALIZE_N(data.base, "base")
       END_KV_SERIALIZE_MAP()
     };
 
diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp
index 9d3b09b68..8822bd378 100644
--- a/src/rpc/daemon_handler.cpp
+++ b/src/rpc/daemon_handler.cpp
@@ -724,12 +724,53 @@ namespace rpc
     res.status = Message::STATUS_OK;
   }
 
-  void DaemonHandler::handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res)
+  void DaemonHandler::handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res)
   {
-    res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks);
+    res.hard_fork_version = m_core.get_blockchain_storage().get_current_hard_fork_version();
+    res.estimated_base_fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks);
+
+    if (res.hard_fork_version < HF_VERSION_PER_BYTE_FEE)
+    {
+       res.size_scale = 1024; // per KiB fee
+       res.fee_mask = 1;
+    }
+    else
+    {
+      res.size_scale = 1; // per byte fee
+      res.fee_mask = Blockchain::get_fee_quantization_mask();
+    }
     res.status = Message::STATUS_OK;
   }
 
+  void DaemonHandler::handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res)
+  {
+    try
+    {
+      res.distributions.reserve(req.amounts.size());
+
+      const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1);
+      for (std::uint64_t amount : req.amounts)
+      {
+        auto data = get_output_distribution(m_core, amount, req.from_height, req_to_height, req.cumulative);
+        if (!data)
+        {
+          res.distributions.clear();
+          res.status = Message::STATUS_FAILED;
+          res.error_details = "Failed to get output distribution";
+          return;
+        }
+        res.distributions.push_back(output_distribution{std::move(*data), amount, req.cumulative});
+      }
+      res.status = Message::STATUS_OK;
+    }
+    catch (const std::exception& e)
+    {
+      res.distributions.clear();
+      res.status = Message::STATUS_FAILED;
+      res.error_details = e.what();
+    }
+  }
+
   bool DaemonHandler::getBlockHeaderByHash(const crypto::hash& hash_in, cryptonote::rpc::BlockHeaderResponse& header)
   {
     block b;
@@ -804,7 +845,8 @@ namespace rpc
       REQ_RESP_TYPES_MACRO(request_type, GetOutputHistogram, req_json, resp_message, handle);
       REQ_RESP_TYPES_MACRO(request_type, GetOutputKeys, req_json, resp_message, handle);
       REQ_RESP_TYPES_MACRO(request_type, GetRPCVersion, req_json, resp_message, handle);
-      REQ_RESP_TYPES_MACRO(request_type, GetPerKBFeeEstimate, req_json, resp_message, handle);
+      REQ_RESP_TYPES_MACRO(request_type, GetFeeEstimate, req_json, resp_message, handle);
+      REQ_RESP_TYPES_MACRO(request_type, GetOutputDistribution, req_json, resp_message, handle);
 
       // if none of the request types matches
       if (resp_message == NULL)
diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h
index 5f9687511..2c8ac3867 100644
--- a/src/rpc/daemon_handler.h
+++ b/src/rpc/daemon_handler.h
@@ -126,7 +126,9 @@ class DaemonHandler : public RpcHandler
 
     void handle(const GetRPCVersion::Request& req, GetRPCVersion::Response& res);
 
-    void handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res);
+    void handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res);
+
+    void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res);
 
     std::string handle(const std::string& request);
 
diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp
index 56f6f6a8c..fa848cff4 100644
--- a/src/rpc/daemon_messages.cpp
+++ b/src/rpc/daemon_messages.cpp
@@ -59,7 +59,8 @@ const char* const HardForkInfo::name = "hard_fork_info";
 const char* const GetOutputHistogram::name = "get_output_histogram";
 const char* const GetOutputKeys::name = "get_output_keys";
 const char* const GetRPCVersion::name = "get_rpc_version";
-const char* const GetPerKBFeeEstimate::name = "get_dynamic_per_kb_fee_estimate";
+const char* const GetFeeEstimate::name = "get_dynamic_fee_estimate";
+const char* const GetOutputDistribution::name = "get_output_distribution";
 
 
 
@@ -825,7 +826,7 @@ void GetRPCVersion::Response::fromJson(rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, version, version);
 }
 
-rapidjson::Value GetPerKBFeeEstimate::Request::toJson(rapidjson::Document& doc) const
+rapidjson::Value GetFeeEstimate::Request::toJson(rapidjson::Document& doc) const
 {
   auto val = Message::toJson(doc);
 
@@ -836,27 +837,70 @@ rapidjson::Value GetPerKBFeeEstimate::Request::toJson(rapidjson::Document& doc)
   return val;
 }
 
-void GetPerKBFeeEstimate::Request::fromJson(rapidjson::Value& val)
+void GetFeeEstimate::Request::fromJson(rapidjson::Value& val)
 {
   GET_FROM_JSON_OBJECT(val, num_grace_blocks, num_grace_blocks);
 }
 
-rapidjson::Value GetPerKBFeeEstimate::Response::toJson(rapidjson::Document& doc) const
+rapidjson::Value GetFeeEstimate::Response::toJson(rapidjson::Document& doc) const
 {
   auto val = Message::toJson(doc);
 
   auto& al = doc.GetAllocator();
 
-  INSERT_INTO_JSON_OBJECT(val, doc, estimated_fee_per_kb, estimated_fee_per_kb);
+  INSERT_INTO_JSON_OBJECT(val, doc, estimated_base_fee, estimated_base_fee);
+  INSERT_INTO_JSON_OBJECT(val, doc, fee_mask, fee_mask);
+  INSERT_INTO_JSON_OBJECT(val, doc, size_scale, size_scale);
+  INSERT_INTO_JSON_OBJECT(val, doc, hard_fork_version, hard_fork_version);
 
   return val;
 }
 
-void GetPerKBFeeEstimate::Response::fromJson(rapidjson::Value& val)
+void GetFeeEstimate::Response::fromJson(rapidjson::Value& val)
 {
-  GET_FROM_JSON_OBJECT(val, estimated_fee_per_kb, estimated_fee_per_kb);
+  GET_FROM_JSON_OBJECT(val, estimated_base_fee, estimated_base_fee);
+  GET_FROM_JSON_OBJECT(val, fee_mask, fee_mask);
+  GET_FROM_JSON_OBJECT(val, size_scale, size_scale);
+  GET_FROM_JSON_OBJECT(val, hard_fork_version, hard_fork_version);
 }
 
+rapidjson::Value GetOutputDistribution::Request::toJson(rapidjson::Document& doc) const
+{
+  auto val = Message::toJson(doc);
+  auto& al = doc.GetAllocator();
+
+  INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts);
+  INSERT_INTO_JSON_OBJECT(val, doc, from_height, from_height);
+  INSERT_INTO_JSON_OBJECT(val, doc, to_height, to_height);
+  INSERT_INTO_JSON_OBJECT(val, doc, cumulative, cumulative);
+
+  return val;
+}
+
+void GetOutputDistribution::Request::fromJson(rapidjson::Value& val)
+{
+  GET_FROM_JSON_OBJECT(val, amounts, amounts);
+  GET_FROM_JSON_OBJECT(val, from_height, from_height);
+  GET_FROM_JSON_OBJECT(val, to_height, to_height);
+  GET_FROM_JSON_OBJECT(val, cumulative, cumulative);
+}
+
+rapidjson::Value GetOutputDistribution::Response::toJson(rapidjson::Document& doc) const
+{
+  auto val = Message::toJson(doc);
+  auto& al = doc.GetAllocator();
+
+  INSERT_INTO_JSON_OBJECT(val, doc, status, status);
+  INSERT_INTO_JSON_OBJECT(val, doc, distributions, distributions);
+
+  return val;
+}
+
+void GetOutputDistribution::Response::fromJson(rapidjson::Value& val)
+{
+  GET_FROM_JSON_OBJECT(val, status, status);
+  GET_FROM_JSON_OBJECT(val, distributions, distributions);
+}
 
 }  // namespace rpc
 
diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h
index 8488bab0b..d2014247c 100644
--- a/src/rpc/daemon_messages.h
+++ b/src/rpc/daemon_messages.h
@@ -28,6 +28,9 @@
 
 #pragma once
 
+#include <unordered_map>
+#include <vector>
+
 #include "message.h"
 #include "cryptonote_protocol/cryptonote_protocol_defs.h"
 #include "rpc/message_data_structs.h"
@@ -62,8 +65,6 @@ class classname \
 #define END_RPC_MESSAGE_RESPONSE };
 #define END_RPC_MESSAGE_CLASS };
 
-#define COMMA() ,
-
 // NOTE: when using a type with multiple template parameters,
 // replace any comma in the template specifier with the macro
 // above, or the preprocessor will eat the comma in a bad way.
@@ -118,7 +119,8 @@ BEGIN_RPC_MESSAGE_CLASS(GetTransactions);
     RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, tx_hashes);
   END_RPC_MESSAGE_REQUEST;
   BEGIN_RPC_MESSAGE_RESPONSE;
-    RPC_MESSAGE_MEMBER(std::unordered_map<crypto::hash COMMA() cryptonote::rpc::transaction_info>, txs);
+    using txes_map = std::unordered_map<crypto::hash, transaction_info>;
+    RPC_MESSAGE_MEMBER(txes_map, txs);
     RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, missed_hashes);
   END_RPC_MESSAGE_RESPONSE;
 END_RPC_MESSAGE_CLASS;
@@ -407,12 +409,27 @@ BEGIN_RPC_MESSAGE_CLASS(GetRPCVersion);
   END_RPC_MESSAGE_RESPONSE;
 END_RPC_MESSAGE_CLASS;
 
-BEGIN_RPC_MESSAGE_CLASS(GetPerKBFeeEstimate);
+BEGIN_RPC_MESSAGE_CLASS(GetFeeEstimate);
   BEGIN_RPC_MESSAGE_REQUEST;
     RPC_MESSAGE_MEMBER(uint64_t, num_grace_blocks);
   END_RPC_MESSAGE_REQUEST;
   BEGIN_RPC_MESSAGE_RESPONSE;
-    RPC_MESSAGE_MEMBER(uint64_t, estimated_fee_per_kb);
+    RPC_MESSAGE_MEMBER(uint64_t, estimated_base_fee);
+    RPC_MESSAGE_MEMBER(uint64_t, fee_mask);
+    RPC_MESSAGE_MEMBER(uint32_t, size_scale);
+    RPC_MESSAGE_MEMBER(uint8_t, hard_fork_version);
+  END_RPC_MESSAGE_RESPONSE;
+END_RPC_MESSAGE_CLASS;
+
+BEGIN_RPC_MESSAGE_CLASS(GetOutputDistribution);
+  BEGIN_RPC_MESSAGE_REQUEST;
+    RPC_MESSAGE_MEMBER(std::vector<uint64_t>, amounts);
+    RPC_MESSAGE_MEMBER(uint64_t, from_height);
+    RPC_MESSAGE_MEMBER(uint64_t, to_height);
+    RPC_MESSAGE_MEMBER(bool, cumulative);
+  END_RPC_MESSAGE_REQUEST;
+  BEGIN_RPC_MESSAGE_RESPONSE;
+    RPC_MESSAGE_MEMBER(std::vector<output_distribution>, distributions);
   END_RPC_MESSAGE_RESPONSE;
 END_RPC_MESSAGE_CLASS;
 
diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h
index cf15ade1c..3b56aff15 100644
--- a/src/rpc/message_data_structs.h
+++ b/src/rpc/message_data_structs.h
@@ -31,6 +31,7 @@
 #include "crypto/hash.h"
 #include "cryptonote_basic/cryptonote_basic.h"
 #include "ringct/rctSigs.h"
+#include "rpc/rpc_handler.h"
 
 #include <unordered_map>
 #include <vector>
@@ -192,6 +193,12 @@ namespace rpc
     uint64_t start_time;
   };
 
+  struct output_distribution
+  {
+    output_distribution_data data;
+    uint64_t amount;
+    bool cumulative;
+  };
 }  // namespace rpc
 
 }  // namespace cryptonote
diff --git a/src/rpc/rpc_handler.cpp b/src/rpc/rpc_handler.cpp
new file mode 100644
index 000000000..d4beb1928
--- /dev/null
+++ b/src/rpc/rpc_handler.cpp
@@ -0,0 +1,69 @@
+
+#include <algorithm>
+#include <boost/thread/locks.hpp>
+#include <boost/thread/mutex.hpp>
+
+#include "cryptonote_core/cryptonote_core.h"
+
+namespace cryptonote
+{
+namespace rpc
+{
+  namespace
+  {
+    output_distribution_data
+      process_distribution(bool cumulative, std::uint64_t start_height, std::vector<std::uint64_t> distribution, std::uint64_t base)
+    {
+      if (!cumulative && !distribution.empty())
+      {
+        for (std::size_t n = distribution.size() - 1; 0 < n; --n)
+          distribution[n] -= distribution[n - 1];
+        distribution[0] -= base;
+      }
+
+      return {std::move(distribution), start_height, base};
+    }
+  }
+
+  boost::optional<output_distribution_data>
+    RpcHandler::get_output_distribution(core& src, std::uint64_t amount, std::uint64_t from_height, std::uint64_t to_height, bool cumulative)
+  {
+      static struct D
+      {
+        boost::mutex mutex;
+        std::vector<std::uint64_t> cached_distribution;
+        std::uint64_t cached_from, cached_to, cached_start_height, cached_base;
+        bool cached;
+        D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {}
+      } d;
+      const boost::unique_lock<boost::mutex> lock(d.mutex);
+
+      if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to == to_height)
+        return process_distribution(cumulative, d.cached_start_height, d.cached_distribution, d.cached_base);
+
+      std::vector<std::uint64_t> distribution;
+      std::uint64_t start_height, base;
+      if (!src.get_output_distribution(amount, from_height, to_height, start_height, distribution, base))
+        return boost::none;
+
+      if (to_height > 0 && to_height >= from_height)
+      {
+        const std::uint64_t offset = std::max(from_height, start_height);
+        if (offset <= to_height && to_height - offset + 1 < distribution.size())
+          distribution.resize(to_height - offset + 1);
+      }
+
+      if (amount == 0)
+      {
+        d.cached_from = from_height;
+        d.cached_to = to_height;
+        d.cached_distribution = distribution;
+        d.cached_start_height = start_height;
+        d.cached_base = base;
+        d.cached = true;
+      }
+
+      return process_distribution(cumulative, start_height, std::move(distribution), base);
+  }
+} // rpc
+} // cryptonote
diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h
index 64bade5a8..3cccef78a 100644
--- a/src/rpc/rpc_handler.h
+++ b/src/rpc/rpc_handler.h
@@ -28,24 +28,35 @@
 
 #pragma once
 
+#include <boost/optional/optional.hpp>
+#include <cstdint>
 #include <string>
+#include <vector>
 
 namespace cryptonote
 {
+class core;
 
 namespace rpc
 {
 
+struct output_distribution_data
+{
+  std::vector<std::uint64_t> distribution;
+  std::uint64_t start_height;
+  std::uint64_t base;
+};
 
 class RpcHandler
 {
   public:
+    RpcHandler() { }
+    virtual ~RpcHandler() { }
 
     virtual std::string handle(const std::string& request) = 0;
 
-    RpcHandler() { }
-
-    virtual ~RpcHandler() { }
+    static boost::optional<output_distribution_data>
+      get_output_distribution(core& src, std::uint64_t amount, std::uint64_t from_height, std::uint64_t to_height, bool cumulative);
 };