RPC and ZeroMQ APIs to support p2pool

Adds the following:

- "get_miner_data" to RPC API
- "json-miner-data" to ZeroMQ subscriber contexts

Both provide the necessary data to create a custom block template. They are used by p2pool.

Data provided:

- major fork version
- current height
- previous block id
- RandomX seed hash
- network difficulty
- median block weight
- coins mined by the network so far
- mineable mempool transactions

5 files changed, 189 insertions, 15 deletions

diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp
index 8d8a68efb..cd0cdad92 100644
--- a/src/rpc/core_rpc_server.cpp
+++ b/src/rpc/core_rpc_server.cpp
@@ -1861,6 +1861,43 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx)
+  {
+    if(!check_core_ready())
+    {
+      error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY;
+      error_resp.message = "Core is busy";
+      return false;
+    }
+
+    crypto::hash prev_id, seed_hash;
+    difficulty_type difficulty;
+
+    std::vector<tx_block_template_backlog_entry> tx_backlog;
+    if (!m_core.get_miner_data(res.major_version, res.height, prev_id, seed_hash, difficulty, res.median_weight, res.already_generated_coins, tx_backlog))
+    {
+      error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
+      error_resp.message = "Internal error: failed to get miner data";
+      LOG_ERROR("Failed to get miner data");
+      return false;
+    }
+
+    res.tx_backlog.clear();
+    res.tx_backlog.reserve(tx_backlog.size());
+
+    for (const auto& entry : tx_backlog)
+    {
+      res.tx_backlog.emplace_back(COMMAND_RPC_GETMINERDATA::response::tx_backlog_entry{string_tools::pod_to_hex(entry.id), entry.weight, entry.fee});
+    }
+
+    res.prev_id = string_tools::pod_to_hex(prev_id);
+    res.seed_hash = string_tools::pod_to_hex(seed_hash);
+    res.difficulty = cryptonote::hex(difficulty);
+
+    res.status = CORE_RPC_STATUS_OK;
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx)
   {
     RPC_TRACKER(add_aux_pow);
diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h
index b21e43ab0..9bc783299 100644
--- a/src/rpc/core_rpc_server.h
+++ b/src/rpc/core_rpc_server.h
@@ -147,6 +147,7 @@ namespace cryptonote
         MAP_JON_RPC_WE("on_getblockhash",        on_getblockhash,               COMMAND_RPC_GETBLOCKHASH)
         MAP_JON_RPC_WE("get_block_template",     on_getblocktemplate,           COMMAND_RPC_GETBLOCKTEMPLATE)
         MAP_JON_RPC_WE("getblocktemplate",       on_getblocktemplate,           COMMAND_RPC_GETBLOCKTEMPLATE)
+        MAP_JON_RPC_WE("get_miner_data",         on_getminerdata,               COMMAND_RPC_GETMINERDATA)
         MAP_JON_RPC_WE("add_aux_pow",            on_add_aux_pow,                COMMAND_RPC_ADD_AUX_POW)
         MAP_JON_RPC_WE("submit_block",           on_submitblock,                COMMAND_RPC_SUBMITBLOCK)
         MAP_JON_RPC_WE("submitblock",            on_submitblock,                COMMAND_RPC_SUBMITBLOCK)
@@ -228,6 +229,7 @@ namespace cryptonote
     bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL);
     bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL);
     bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL);
+    bool on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL);
     bool on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL);
     bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL);
     bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL);
diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h
index ff8c98b98..1dbfc83a7 100644
--- a/src/rpc/core_rpc_server_commands_defs.h
+++ b/src/rpc/core_rpc_server_commands_defs.h
@@ -88,7 +88,7 @@ namespace cryptonote
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define CORE_RPC_VERSION_MAJOR 3
-#define CORE_RPC_VERSION_MINOR 7
+#define CORE_RPC_VERSION_MINOR 8
 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
 
@@ -940,6 +940,56 @@ namespace cryptonote
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
+  struct COMMAND_RPC_GETMINERDATA
+  {
+    struct request_t: public rpc_request_base
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE_PARENT(rpc_request_base)
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<request_t> request;
+
+    struct response_t: public rpc_response_base
+    {
+      uint8_t major_version;
+      uint64_t height;
+      std::string prev_id;
+      std::string seed_hash;
+      std::string difficulty;
+      uint64_t median_weight;
+      uint64_t already_generated_coins;
+
+      struct tx_backlog_entry
+      {
+        std::string id;
+        uint64_t weight;
+        uint64_t fee;
+
+        BEGIN_KV_SERIALIZE_MAP()
+          KV_SERIALIZE(id)
+          KV_SERIALIZE(weight)
+          KV_SERIALIZE(fee)
+        END_KV_SERIALIZE_MAP()
+      };
+
+      std::vector<tx_backlog_entry> tx_backlog;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE_PARENT(rpc_response_base)
+        KV_SERIALIZE(major_version)
+        KV_SERIALIZE(height)
+        KV_SERIALIZE(prev_id)
+        KV_SERIALIZE(seed_hash)
+        KV_SERIALIZE(difficulty)
+        KV_SERIALIZE(median_weight)
+        KV_SERIALIZE(already_generated_coins)
+        KV_SERIALIZE(tx_backlog)
+      END_KV_SERIALIZE_MAP()
+    };
+    typedef epee::misc_utils::struct_init<response_t> response;
+  };
+
   struct COMMAND_RPC_ADD_AUX_POW
   {
     struct aux_pow_t
diff --git a/src/rpc/zmq_pub.cpp b/src/rpc/zmq_pub.cpp
index eac530968..074b55207 100644
--- a/src/rpc/zmq_pub.cpp
+++ b/src/rpc/zmq_pub.cpp
@@ -48,6 +48,8 @@
 #include "cryptonote_basic/events.h"
 #include "misc_log_ex.h"
 #include "serialization/json_object.h"
+#include "ringct/rctTypes.h"
+#include "cryptonote_core/cryptonote_tx_utils.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net.zmq"
@@ -57,6 +59,7 @@ namespace
   constexpr const char txpool_signal[] = "tx_signal";
 
   using chain_writer =  void(epee::byte_stream&, std::uint64_t, epee::span<const cryptonote::block>);
+  using miner_writer =  void(epee::byte_stream&, uint8_t, uint64_t, const crypto::hash&, const crypto::hash&, cryptonote::difficulty_type, uint64_t, uint64_t, const std::vector<cryptonote::tx_block_template_backlog_entry>&);
   using txpool_writer = void(epee::byte_stream&, epee::span<const cryptonote::txpool_event>);
 
   template<typename F>
@@ -116,13 +119,30 @@ namespace
     const epee::span<const cryptonote::block> blocks;
   };
 
+  //! Object for miner data serialization
+  struct miner_data
+  {
+    uint8_t major_version;
+    uint64_t height;
+    const crypto::hash& prev_id;
+    const crypto::hash& seed_hash;
+    cryptonote::difficulty_type diff;
+    uint64_t median_weight;
+    uint64_t already_generated_coins;
+    const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog;
+  };
+
   //! Object for "minimal" tx serialization
   struct minimal_txpool
   {
     const cryptonote::transaction& tx;
+    crypto::hash hash;
+    uint64_t blob_size;
+    uint64_t weight;
+    uint64_t fee;
   };
 
-  void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain self)
+  void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain& self)
   {
     namespace adapt = boost::adaptors;
 
@@ -143,19 +163,27 @@ namespace
     dest.EndObject();
   }
 
-  void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool self)
+  void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const miner_data& self)
   {
-    crypto::hash id{};
-    std::size_t blob_size = 0;
-    if (!get_transaction_hash(self.tx, id, blob_size))
-    {
-      MERROR("ZMQ/Pub failure: get_transaction_hash");
-      return;
-    }
+    dest.StartObject();
+    INSERT_INTO_JSON_OBJECT(dest, major_version, self.major_version);
+    INSERT_INTO_JSON_OBJECT(dest, height, self.height);
+    INSERT_INTO_JSON_OBJECT(dest, prev_id, self.prev_id);
+    INSERT_INTO_JSON_OBJECT(dest, seed_hash, self.seed_hash);
+    INSERT_INTO_JSON_OBJECT(dest, difficulty, cryptonote::hex(self.diff));
+    INSERT_INTO_JSON_OBJECT(dest, median_weight, self.median_weight);
+    INSERT_INTO_JSON_OBJECT(dest, already_generated_coins, self.already_generated_coins);
+    INSERT_INTO_JSON_OBJECT(dest, tx_backlog, self.tx_backlog);
+    dest.EndObject();
+  }
 
+  void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool& self)
+  {
     dest.StartObject();
-    INSERT_INTO_JSON_OBJECT(dest, id, id);
-    INSERT_INTO_JSON_OBJECT(dest, blob_size, blob_size);
+    INSERT_INTO_JSON_OBJECT(dest, id, self.hash);
+    INSERT_INTO_JSON_OBJECT(dest, blob_size, self.blob_size);
+    INSERT_INTO_JSON_OBJECT(dest, weight, self.weight);
+    INSERT_INTO_JSON_OBJECT(dest, fee, self.fee);
     dest.EndObject();
   }
 
@@ -169,6 +197,11 @@ namespace
     json_pub(buf, minimal_chain{height, blocks});
   }
 
+  void json_miner_data(epee::byte_stream& buf, uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, cryptonote::difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog)
+  {
+    json_pub(buf, miner_data{major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog});
+  }
+
   // boost::adaptors are in place "views" - no copy/move takes place
   // moving transactions (via sort, etc.), is expensive!
 
@@ -187,7 +220,7 @@ namespace
     namespace adapt = boost::adaptors;
     const auto to_minimal_tx = [](const cryptonote::txpool_event& event)
     {
-      return minimal_txpool{event.tx};
+      return minimal_txpool{event.tx, event.hash, event.blob_size, event.weight, cryptonote::get_tx_fee(event.tx)};
     };
     json_pub(buf, (txes | adapt::filtered(is_valid{}) | adapt::transformed(to_minimal_tx)));
   }
@@ -198,6 +231,11 @@ namespace
     {u8"json-minimal-chain_main", json_minimal_chain}
   }};
 
+  constexpr const std::array<context<miner_writer>, 1> miner_contexts =
+  {{
+    {u8"json-full-miner_data", json_miner_data},
+  }};
+
   constexpr const std::array<context<txpool_writer>, 2> txpool_contexts =
   {{
     {u8"json-full-txpool_add", json_full_txpool},
@@ -321,6 +359,7 @@ namespace cryptonote { namespace listener
 zmq_pub::zmq_pub(void* context)
   : relay_(),
     chain_subs_{{0}},
+    miner_subs_{{0}},
     txpool_subs_{{0}},
     sync_()
 {
@@ -328,6 +367,7 @@ zmq_pub::zmq_pub(void* context)
     throw std::logic_error{"ZMQ context cannot be NULL"};
 
   verify_sorted(chain_contexts, "chain_contexts");
+  verify_sorted(miner_contexts, "miner_contexts");
   verify_sorted(txpool_contexts, "txpool_contexts");
 
   relay_.reset(zmq_socket(context, ZMQ_PAIR));
@@ -348,22 +388,25 @@ bool zmq_pub::sub_request(boost::string_ref message)
     message.remove_prefix(1);
 
     const auto chain_range = get_range(chain_contexts, message);
+    const auto miner_range = get_range(miner_contexts, message);
     const auto txpool_range = get_range(txpool_contexts, message);
 
-    if (!chain_range.empty() || !txpool_range.empty())
+    if (!chain_range.empty() || !miner_range.empty() || !txpool_range.empty())
     {
       MDEBUG("Client " << (tag ? "subscribed" : "unsubscribed") << " to " <<
-             chain_range.size() << " chain topic(s) and " << txpool_range.size() << " txpool topic(s)");
+             chain_range.size() << " chain topic(s), " << miner_range.size() << " miner topic(s) and " << txpool_range.size() << " txpool topic(s)");
 
       const boost::lock_guard<boost::mutex> lock{sync_};
       switch (tag)
       {
       case 0:
         remove_subscriptions(chain_subs_, chain_range, chain_contexts.begin());
+        remove_subscriptions(miner_subs_, miner_range, miner_contexts.begin());
         remove_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin());
         return true;
       case 1:
         add_subscriptions(chain_subs_, chain_range, chain_contexts.begin());
+        add_subscriptions(miner_subs_, miner_range, miner_contexts.begin());
         add_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin());
         return true;
       default:
@@ -436,6 +479,25 @@ std::size_t zmq_pub::send_chain_main(const std::uint64_t height, const epee::spa
   return 0;
 }
 
+std::size_t zmq_pub::send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog)
+{
+  boost::unique_lock<boost::mutex> guard{sync_};
+
+  const auto subs_copy = miner_subs_;
+  guard.unlock();
+
+  for (const std::size_t sub : subs_copy)
+  {
+    if (sub)
+    {
+        auto messages = make_pubs(subs_copy, miner_contexts, major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog);
+        guard.lock();
+        return send_messages(relay_.get(), messages);
+    }
+  }
+  return 0;
+}
+
 std::size_t zmq_pub::send_txpool_add(std::vector<txpool_event> txes)
 {
   if (txes.empty())
@@ -466,6 +528,15 @@ void zmq_pub::chain_main::operator()(const std::uint64_t height, epee::span<cons
     MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed");
 }
 
+void zmq_pub::miner_data::operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const
+{
+  const std::shared_ptr<zmq_pub> self = self_.lock();
+  if (self)
+    self->send_miner_data(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog);
+  else
+    MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed");
+}
+
 void zmq_pub::txpool_add::operator()(std::vector<cryptonote::txpool_event> txes) const
 {
   const std::shared_ptr<zmq_pub> self = self_.lock();
diff --git a/src/rpc/zmq_pub.h b/src/rpc/zmq_pub.h
index 02e6b8103..c636e1d7b 100644
--- a/src/rpc/zmq_pub.h
+++ b/src/rpc/zmq_pub.h
@@ -39,6 +39,7 @@
 #include "cryptonote_basic/fwd.h"
 #include "net/zmq.h"
 #include "span.h"
+#include "cryptonote_basic/difficulty.h"
 
 namespace cryptonote { namespace listener
 {
@@ -59,6 +60,7 @@ class zmq_pub
     net::zmq::socket relay_;
     std::deque<std::vector<txpool_event>> txes_;
     std::array<std::size_t, 2> chain_subs_;
+    std::array<std::size_t, 1> miner_subs_;
     std::array<std::size_t, 2> txpool_subs_;
     boost::mutex sync_; //!< Synchronizes counts in `*_subs_` arrays.
 
@@ -88,6 +90,11 @@ class zmq_pub
         \return Number of ZMQ messages sent to relay. */
     std::size_t send_chain_main(std::uint64_t height, epee::span<const cryptonote::block> blocks);
 
+    /*! Send a `ZMQ_PUB` notification for a new miner data.
+        Thread-safe.
+        \return Number of ZMQ messages sent to relay. */
+    std::size_t send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog);
+
     /*! Send a `ZMQ_PUB` notification for new tx(es) being added to the local
         pool. Thread-safe.
         \return Number of ZMQ messages sent to relay. */
@@ -100,6 +107,13 @@ class zmq_pub
       void operator()(std::uint64_t height, epee::span<const cryptonote::block> blocks) const;
     };
 
+    //! Callable for `send_miner_data` with weak ownership to `zmq_pub` object.
+    struct miner_data
+    {
+      std::weak_ptr<zmq_pub> self_;
+      void operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const;
+    };
+
     //! Callable for `send_txpool_add` with weak ownership to `zmq_pub` object.
     struct txpool_add
     {