monerod can now sync from pruned blocks

If the peer (whether pruned or not itself) supports sending pruned blocks
to syncing nodes, the pruned version will be sent along with the hash
of the pruned data and the block weight. The original tx hashes can be
reconstructed from the pruned txes and theur prunable data hash. Those
hashes and the block weights are hashes and checked against the set of
precompiled hashes, ensuring the data we received is the original data.
It is currently not possible to use this system when not using the set
of precompiled hashes, since block weights can not otherwise be checked
for validity.

This is off by default for now, and is enabled by --sync-pruned-blocks

3 files changed, 92 insertions, 17 deletions

diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h
index 51076e8c0..5e8f6685d 100644
--- a/src/cryptonote_basic/connection_context.h
+++ b/src/cryptonote_basic/connection_context.h
@@ -55,7 +55,7 @@ namespace cryptonote
     };
 
     state m_state;
-    std::vector<crypto::hash> m_needed_objects;
+    std::vector<std::pair<crypto::hash, uint64_t>> m_needed_objects;
     std::unordered_set<crypto::hash> m_requested_objects;
     uint64_t m_remote_blockchain_height;
     uint64_t m_last_response_height;
diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h
index 055c4a22b..e2286ae8c 100644
--- a/src/cryptonote_basic/cryptonote_basic.h
+++ b/src/cryptonote_basic/cryptonote_basic.h
@@ -195,6 +195,7 @@ namespace cryptonote
   private:
     // hash cash
     mutable std::atomic<bool> hash_valid;
+    mutable std::atomic<bool> prunable_hash_valid;
     mutable std::atomic<bool> blob_size_valid;
 
   public:
@@ -203,6 +204,7 @@ namespace cryptonote
 
     // hash cash
     mutable crypto::hash hash;
+    mutable crypto::hash prunable_hash;
     mutable size_t blob_size;
 
     bool pruned;
@@ -211,22 +213,26 @@ namespace cryptonote
     std::atomic<unsigned int> prefix_size;
 
     transaction();
-    transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned), unprunable_size(t.unprunable_size.load()), prefix_size(t.prefix_size.load()) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } }
-    transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; unprunable_size = t.unprunable_size.load(); prefix_size = t.prefix_size.load(); return *this; }
+    transaction(const transaction &t);
+    transaction &operator=(const transaction &t);
     virtual ~transaction();
     void set_null();
     void invalidate_hashes();
     bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
     void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
+    bool is_prunable_hash_valid() const { return prunable_hash_valid.load(std::memory_order_acquire); }
+    void set_prunable_hash_valid(bool v) const { prunable_hash_valid.store(v,std::memory_order_release); }
     bool is_blob_size_valid() const { return blob_size_valid.load(std::memory_order_acquire); }
     void set_blob_size_valid(bool v) const { blob_size_valid.store(v,std::memory_order_release); }
-    void set_hash(const crypto::hash &h) { hash = h; set_hash_valid(true); }
-    void set_blob_size(size_t sz) { blob_size = sz; set_blob_size_valid(true); }
+    void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); }
+    void set_prunable_hash(const crypto::hash &h) const { prunable_hash = h; set_prunable_hash_valid(true); }
+    void set_blob_size(size_t sz) const { blob_size = sz; set_blob_size_valid(true); }
 
     BEGIN_SERIALIZE_OBJECT()
       if (!typename Archive<W>::is_saving())
       {
         set_hash_valid(false);
+        set_prunable_hash_valid(false);
         set_blob_size_valid(false);
       }
 
@@ -327,6 +333,63 @@ namespace cryptonote
     static size_t get_signature_size(const txin_v& tx_in);
   };
 
+  inline transaction::transaction(const transaction &t):
+    transaction_prefix(t),
+    hash_valid(false),
+    prunable_hash_valid(false),
+    blob_size_valid(false),
+    signatures(t.signatures),
+    rct_signatures(t.rct_signatures),
+    pruned(t.pruned),
+    unprunable_size(t.unprunable_size.load()),
+    prefix_size(t.prefix_size.load())
+  {
+    if (t.is_hash_valid())
+    {
+      hash = t.hash;
+      set_hash_valid(true);
+    }
+    if (t.is_blob_size_valid())
+    {
+      blob_size = t.blob_size;
+      set_blob_size_valid(true);
+    }
+    if (t.is_prunable_hash_valid())
+    {
+      prunable_hash = t.prunable_hash;
+      set_prunable_hash_valid(true);
+    }
+  }
+
+  inline transaction &transaction::operator=(const transaction &t)
+  {
+    transaction_prefix::operator=(t);
+
+    set_hash_valid(false);
+    set_prunable_hash_valid(false);
+    set_blob_size_valid(false);
+    signatures = t.signatures;
+    rct_signatures = t.rct_signatures;
+    if (t.is_hash_valid())
+    {
+      hash = t.hash;
+      set_hash_valid(true);
+    }
+    if (t.is_prunable_hash_valid())
+    {
+      prunable_hash = t.prunable_hash;
+      set_prunable_hash_valid(true);
+    }
+    if (t.is_blob_size_valid())
+    {
+      blob_size = t.blob_size;
+      set_blob_size_valid(true);
+    }
+    pruned = t.pruned;
+    unprunable_size = t.unprunable_size.load();
+    prefix_size = t.prefix_size.load();
+    return *this;
+  }
 
   inline
   transaction::transaction()
@@ -346,6 +409,7 @@ namespace cryptonote
     signatures.clear();
     rct_signatures.type = rct::RCTTypeNull;
     set_hash_valid(false);
+    set_prunable_hash_valid(false);
     set_blob_size_valid(false);
     pruned = false;
     unprunable_size = 0;
@@ -356,6 +420,7 @@ namespace cryptonote
   void transaction::invalidate_hashes()
   {
     set_hash_valid(false);
+    set_prunable_hash_valid(false);
     set_blob_size_valid(false);
   }
 
@@ -408,6 +473,7 @@ namespace cryptonote
     void invalidate_hashes() { set_hash_valid(false); }
     bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
     void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
+    void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); }
 
     transaction miner_tx;
     std::vector<crypto::hash> tx_hashes;
diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp
index 7d7de416d..17f861e9a 100644
--- a/src/cryptonote_basic/cryptonote_format_utils.cpp
+++ b/src/cryptonote_basic/cryptonote_format_utils.cpp
@@ -1011,7 +1011,19 @@ namespace cryptonote
   crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata)
   {
     crypto::hash res;
+    if (t.is_prunable_hash_valid())
+    {
+#ifdef ENABLE_HASH_CASH_INTEGRITY_CHECK
+      CHECK_AND_ASSERT_THROW_MES(!calculate_transaction_prunable_hash(t, blobdata, res) || t.hash == res, "tx hash cash integrity failure");
+#endif
+      res = t.prunable_hash;
+      ++tx_hashes_cached_count;
+      return res;
+    }
+
+    ++tx_hashes_calculated_count;
     CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, blobdata, res), "Failed to calculate tx prunable hash");
+    t.set_prunable_hash(res);
     return res;
   }
   //---------------------------------------------------------------
@@ -1047,11 +1059,14 @@ namespace cryptonote
 
     // the tx hash is the hash of the 3 hashes
     crypto::hash res = cn_fast_hash(hashes, sizeof(hashes));
+    t.set_hash(res);
     return res;
   }
   //---------------------------------------------------------------
   bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
   {
+    CHECK_AND_ASSERT_MES(!t.pruned, false, "Cannot calculate the hash of a pruned transaction");
+
     // v1 transactions hash the entire blob
     if (t.version == 1)
     {
@@ -1091,8 +1106,7 @@ namespace cryptonote
     {
       if (!t.is_blob_size_valid())
       {
-        t.blob_size = blob.size();
-        t.set_blob_size_valid(true);
+        t.set_blob_size(blob.size());
       }
       *blob_size = t.blob_size;
     }
@@ -1112,8 +1126,7 @@ namespace cryptonote
       {
         if (!t.is_blob_size_valid())
         {
-          t.blob_size = get_object_blobsize(t);
-          t.set_blob_size_valid(true);
+          t.set_blob_size(get_object_blobsize(t));
         }
         *blob_size = t.blob_size;
       }
@@ -1124,12 +1137,10 @@ namespace cryptonote
     bool ret = calculate_transaction_hash(t, res, blob_size);
     if (!ret)
       return false;
-    t.hash = res;
-    t.set_hash_valid(true);
+    t.set_hash(res);
     if (blob_size)
     {
-      t.blob_size = *blob_size;
-      t.set_blob_size_valid(true);
+      t.set_blob_size(*blob_size);
     }
     return true;
   }
@@ -1206,8 +1217,7 @@ namespace cryptonote
     bool ret = calculate_block_hash(b, res);
     if (!ret)
       return false;
-    b.hash = res;
-    b.set_hash_valid(true);
+    b.set_hash(res);
     return true;
   }
   //---------------------------------------------------------------
@@ -1273,8 +1283,7 @@ namespace cryptonote
     {
       calculate_block_hash(b, *block_hash, &b_blob);
       ++block_hashes_calculated_count;
-      b.hash = *block_hash;
-      b.set_hash_valid(true);
+      b.set_hash(*block_hash);
     }
     return true;
   }