ringct: do not serialize what can be reconstructed

The mixRing (output keys and commitments) and II fields (key images)
can be reconstructed from vin data.
This saves some modest amount of space in the tx.

6 files changed, 132 insertions, 50 deletions

diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp
index 0d7294ffd..82bc6c0d4 100644
--- a/src/cryptonote_core/blockchain.cpp
+++ b/src/cryptonote_core/blockchain.cpp
@@ -2461,6 +2461,13 @@ bool Blockchain::check_tx_inputs(const transaction& tx, tx_verification_context
   else
   {
     // from version 2, check ringct signatures
+    rct::ctkeyM reconstructed_mixRing;
+    rct::keyV reconstructed_II;
+
+    // if the tx already has a non empty mixRing and/or II, use them,
+    // else reconstruct them
+    const rct::ctkeyM &mixRing = tx.rct_signatures.mixRing.empty() ? reconstructed_mixRing : tx.rct_signatures.mixRing;
+    const rct::keyV &II = tx.rct_signatures.MG.II.size() == 1 ? reconstructed_II : tx.rct_signatures.MG.II;
 
     // RCT needs the same mixin for all inputs
     for (size_t n = 1; n < pubkeys.size(); ++n)
@@ -2472,35 +2479,74 @@ bool Blockchain::check_tx_inputs(const transaction& tx, tx_verification_context
       }
     }
 
-    bool size_matches = true;
-    for (size_t i = 0; i < pubkeys.size(); ++i)
-      size_matches &= pubkeys[i].size() == tx.rct_signatures.mixRing.size();
-    for (size_t i = 0; i < tx.rct_signatures.mixRing.size(); ++i)
-      size_matches &= pubkeys.size() == tx.rct_signatures.mixRing[i].size();
-    if (!size_matches)
+    if (tx.rct_signatures.mixRing.empty())
     {
-      LOG_PRINT_L1("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
-      return false;
+      reconstructed_mixRing.resize(pubkeys[0].size());
+      for (size_t n = 0; n < pubkeys.size(); ++n)
+      {
+        for (size_t m = 0; m < pubkeys[n].size(); ++m)
+        {
+          reconstructed_mixRing[m].push_back(pubkeys[n][m]);
+        }
+      }
     }
 
-    for (size_t n = 0; n < pubkeys.size(); ++n)
+    if (tx.rct_signatures.MG.II.size() == 1)
     {
-      for (size_t m = 0; m < pubkeys[n].size(); ++m)
+      reconstructed_II.resize(tx.vin.size());
+      for (size_t n = 0; n < tx.vin.size(); ++n)
       {
-        if (pubkeys[n][m].dest != rct::rct2pk(tx.rct_signatures.mixRing[m][n].dest))
-        {
-          LOG_PRINT_L1("Failed to check ringct signatures: mismatched pubkey at vin " << n << ", index " << m);
-          return false;
-        }
-        if (pubkeys[n][m].mask != rct::rct2pk(tx.rct_signatures.mixRing[m][n].mask))
+        reconstructed_II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image);
+      }
+      reconstructed_II.push_back(tx.rct_signatures.MG.II.back());
+    }
+
+    // check all this, either recontructed (so should really pass), or not
+    {
+      bool size_matches = true;
+      for (size_t i = 0; i < pubkeys.size(); ++i)
+        size_matches &= pubkeys[i].size() == mixRing.size();
+      for (size_t i = 0; i < tx.rct_signatures.mixRing.size(); ++i)
+        size_matches &= pubkeys.size() == mixRing[i].size();
+      if (!size_matches)
+      {
+        LOG_PRINT_L1("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
+        return false;
+      }
+
+      for (size_t n = 0; n < pubkeys.size(); ++n)
+      {
+        for (size_t m = 0; m < pubkeys[n].size(); ++m)
         {
-          LOG_PRINT_L1("Failed to check ringct signatures: mismatched commitment at vin " << n << ", index " << m);
-          return false;
+          if (pubkeys[n][m].dest != rct::rct2pk(mixRing[m][n].dest))
+          {
+            LOG_PRINT_L1("Failed to check ringct signatures: mismatched pubkey at vin " << n << ", index " << m);
+            return false;
+          }
+          if (pubkeys[n][m].mask != rct::rct2pk(mixRing[m][n].mask))
+          {
+            LOG_PRINT_L1("Failed to check ringct signatures: mismatched commitment at vin " << n << ", index " << m);
+            return false;
+          }
         }
       }
     }
 
-    if (!rct::verRct(tx.rct_signatures))
+    if (II.size() != 1 + tx.vin.size())
+    {
+      LOG_PRINT_L1("Failed to check ringct signatures: mismatched II/vin sizes");
+      return false;
+    }
+    for (size_t n = 0; n < tx.vin.size(); ++n)
+    {
+      if (memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &II[n], 32))
+      {
+        LOG_PRINT_L1("Failed to check ringct signatures: mismatched II/vin sizes");
+        return false;
+      }
+    }
+
+    if (!rct::verRct(tx.rct_signatures, mixRing, II, rct::hash2rct(tx_prefix_hash)))
     {
       LOG_PRINT_L1("Failed to check ringct signatures!");
       return false;
@@ -2613,7 +2659,8 @@ bool Blockchain::check_tx_input(size_t tx_version, const txin_to_key& txin, cons
   }
   else
   {
-    CHECK_AND_ASSERT_MES(rct_signatures.mixRing.size() == output_keys.size(), false, "internal error: tx rct signatures count=" << sig.size() << " mismatch with outputs keys count for inputs=" << output_keys.size());
+    // rct signatures may be empty (and will be reconstructed later in the caller if so)
+    CHECK_AND_ASSERT_MES(rct_signatures.mixRing.empty() || rct_signatures.mixRing.size() == output_keys.size(), false, "internal error: tx rct signatures count=" << sig.size() << " mismatch with outputs keys count for inputs=" << output_keys.size());
   }
   return true;
 }
diff --git a/src/cryptonote_core/cryptonote_boost_serialization.h b/src/cryptonote_core/cryptonote_boost_serialization.h
index 6eeb66ec8..ccdd55883 100644
--- a/src/cryptonote_core/cryptonote_boost_serialization.h
+++ b/src/cryptonote_core/cryptonote_boost_serialization.h
@@ -37,6 +37,8 @@
 #include <boost/serialization/map.hpp>
 #include <boost/foreach.hpp>
 #include <boost/serialization/is_bitwise_serializable.hpp>
+#include <boost/archive/binary_iarchive.hpp>
+#include <boost/archive/binary_oarchive.hpp>
 #include "cryptonote_basic.h"
 #include "common/unordered_containers_boost_serialization.h"
 #include "crypto/crypto.h"
@@ -196,12 +198,19 @@ namespace boost
     a & x.s;
   }
 
-  template <class Archive>
-  inline void serialize(Archive &a, rct::mgSig &x, const boost::serialization::version_type ver)
+  inline void serialize(boost::archive::binary_iarchive &a, rct::mgSig &x, const boost::serialization::version_type ver)
+  {
+    a & x.ss;
+    a & x.cc;
+    x.II.resize(1);
+    a & x.II[0];
+  }
+
+  inline void serialize(boost::archive::binary_oarchive &a, rct::mgSig &x, const boost::serialization::version_type ver)
   {
     a & x.ss;
     a & x.cc;
-    a & x.II;
+    a & x.II.back();
   }
 
   template <class Archive>
@@ -217,10 +226,11 @@ namespace boost
   {
     a & x.rangeSigs;
     a & x.MG;
-    a & x.mixRing;
+    // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets
     a & x.ecdhInfo;
     a & x.outPk;
     a & x.txnFee;
+    // a & x.bash_hash; bash_hash is not serialized, as it can be reconstructed from the tx data
   }
 }
 }
diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp
index 89adb2a77..917ac54cd 100644
--- a/src/ringct/rctSigs.cpp
+++ b/src/ringct/rctSigs.cpp
@@ -226,7 +226,7 @@ namespace rct {
     // Gen creates a signature which proves that for some column in the keymatrix "pk"
     //   the signer knows a secret key for each row in that column
     // Ver verifies that the MG sig was created correctly            
-    bool MLSAG_Ver(key message, const keyM & pk, const mgSig & rv) {
+    bool MLSAG_Ver(key message, const keyM & pk, const mgSig & rv, const keyV &II) {
 
         size_t cols = pk.size();
         CHECK_AND_ASSERT_MES(cols >= 2, false, "Error! What is c if cols = 1!");
@@ -235,7 +235,7 @@ namespace rct {
         for (size_t i = 1; i < cols; ++i) {
           CHECK_AND_ASSERT_MES(pk[i].size() == rows, false, "pk is not rectangular");
         }
-        CHECK_AND_ASSERT_MES(rv.II.size() == rows, false, "Bad rv.II size");
+        CHECK_AND_ASSERT_MES(II.size() == rows, false, "Bad II size");
         CHECK_AND_ASSERT_MES(rv.ss.size() == cols, false, "Bad rv.ss size");
         for (size_t i = 0; i < cols; ++i) {
           CHECK_AND_ASSERT_MES(rv.ss[i].size() == rows, false, "rv.ss is not rectangular");
@@ -246,7 +246,7 @@ namespace rct {
         key c_old = copy(rv.cc);
         vector<geDsmp> Ip(rows);
         for (i= 0 ; i< rows ; i++) {
-            precomp(Ip[i].k, rv.II[i]);
+            precomp(Ip[i].k, II[i]);
         }
         unsigned char m2[128]; 
         memcpy(m2, message.bytes, 32);
@@ -334,7 +334,7 @@ namespace rct {
     //   this shows that sum inputs = sum outputs
     //Ver:    
     //   verifies the above sig is created corretly
-    mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const ctkeyV &outSk, const ctkeyV & outPk, unsigned int index, key txnFeeKey) {
+    mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const ctkeyV &outSk, const ctkeyV & outPk, unsigned int index, key txnFeeKey, const key &base_hash) {
         mgSig mg;
         //setup vars
         size_t cols = pubs.size();
@@ -378,7 +378,9 @@ namespace rct {
         for (size_t j = 0; j < outPk.size(); j++) {
             sc_sub(sk[rows].bytes, sk[rows].bytes, outSk[j].mask.bytes); //subtract output masks in last row..
         }
-        key message = cn_fast_hash(outPk);
+        ctkeyV signed_data = outPk;
+        signed_data.push_back(ctkey({base_hash, identity()}));
+        key message = cn_fast_hash(signed_data);
         return MLSAG_Gen(message, M, sk, index);
     }
 
@@ -391,7 +393,7 @@ namespace rct {
     //   this shows that sum inputs = sum outputs
     //Ver:    
     //   verifies the above sig is created corretly
-    bool verRctMG(mgSig mg, const ctkeyM & pubs, const ctkeyV & outPk, key txnFeeKey) {
+    bool verRctMG(mgSig mg, const keyV &II, const ctkeyM & pubs, const ctkeyV & outPk, key txnFeeKey, const key &base_hash) {
         //setup vars
         size_t cols = pubs.size();
         CHECK_AND_ASSERT_MES(cols >= 1, false, "Empty pubs");
@@ -422,10 +424,12 @@ namespace rct {
             //subtract txn fee output in last row
             subKeys(M[i][rows], M[i][rows], txnFeeKey);
         }
-        key message = cn_fast_hash(outPk);
+        ctkeyV signed_data = outPk;
+        signed_data.push_back(ctkey({base_hash, identity()}));
+        key message = cn_fast_hash(signed_data);
         DP("message:");
         DP(message);
-        return MLSAG_Ver(message, M, mg);
+        return MLSAG_Ver(message, M, mg, II);
     }
 
     //These functions get keys from blockchain
@@ -470,7 +474,7 @@ namespace rct {
     //   must know the destination private key to find the correct amount, else will return a random number
     //   Note: For txn fees, the last index in the amounts vector should contain that
     //   Thus the amounts vector will be "one" longer than the destinations vectort
-    rctSig genRct(const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> amounts, const ctkeyM &mixRing, unsigned int index) {
+    rctSig genRct(const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> amounts, const ctkeyM &mixRing, const key &base_hash, unsigned int index) {
         CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations");
         CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing");
         for (size_t n = 0; n < mixRing.size(); ++n) {
@@ -513,15 +517,16 @@ namespace rct {
         key txnFeeKey = scalarmultH(d2h(rv.txnFee));
 
         rv.mixRing = mixRing;
-        rv.MG = proveRctMG(rv.mixRing, inSk, outSk, rv.outPk, index, txnFeeKey);
+        rv.base_hash = base_hash;
+        rv.MG = proveRctMG(rv.mixRing, inSk, outSk, rv.outPk, index, txnFeeKey, base_hash);
         return rv;
     }
 
-    rctSig genRct(const ctkeyV & inSk, const ctkeyV  & inPk, const keyV & destinations, const vector<xmr_amount> amounts, const int mixin) {
+    rctSig genRct(const ctkeyV & inSk, const ctkeyV  & inPk, const keyV & destinations, const vector<xmr_amount> amounts, const key &base_hash, const int mixin) {
         unsigned int index;
         ctkeyM mixRing;
         tie(mixRing, index) = populateFromBlockchain(inPk, mixin);
-        return genRct(inSk, destinations, amounts, mixRing, index);
+        return genRct(inSk, destinations, amounts, mixRing, base_hash, index);
     }
     
     //RingCT protocol
@@ -534,7 +539,7 @@ namespace rct {
     //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1)
     //   uses the attached ecdh info to find the amounts represented by each output commitment 
     //   must know the destination private key to find the correct amount, else will return a random number    
-    bool verRct(const rctSig & rv) {
+    bool verRct(const rctSig & rv, const ctkeyM &mixRing, const keyV &II, const key &base_hash) {
         CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.rangeSigs.size(), false, "Mismatched sizes of rv.outPk and rv.rangeSigs");
         CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of rv.outPk and rv.ecdhInfo");
 
@@ -552,7 +557,7 @@ namespace rct {
           }
           //compute txn fee
           key txnFeeKey = scalarmultH(d2h(rv.txnFee));
-          bool mgVerd = verRctMG(rv.MG, rv.mixRing, rv.outPk, txnFeeKey);
+          bool mgVerd = verRctMG(rv.MG, II, mixRing, rv.outPk, txnFeeKey, base_hash);
           DP("mg sig verified?");
           DP(mgVerd);
 
@@ -563,6 +568,9 @@ namespace rct {
           return false;
         }
     }
+    bool verRct(const rctSig & rv) {
+        return verRct(rv, rv.mixRing, rv.MG.II, rv.base_hash);
+    }
     
     //RingCT protocol
     //genRct: 
diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h
index d5c036910..f87312fa8 100644
--- a/src/ringct/rctSigs.h
+++ b/src/ringct/rctSigs.h
@@ -91,7 +91,7 @@ namespace rct {
     // Ver verifies that the MG sig was created correctly
     keyV keyImageV(const keyV &xx);
     mgSig MLSAG_Gen(key message, const keyM & pk, const keyV & xx, const unsigned int index);
-    bool MLSAG_Ver(key message, const keyM &pk, const mgSig &sig);
+    bool MLSAG_Ver(key message, const keyM &pk, const mgSig &sig, const keyV &II);
     //mgSig MLSAG_Gen_Old(const keyM & pk, const keyV & xx, const int index);
 
     //proveRange and verRange
@@ -112,8 +112,8 @@ namespace rct {
     //   this shows that sum inputs = sum outputs
     //Ver:
     //   verifies the above sig is created corretly
-    mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const keyV &outMasks, const ctkeyV & outPk, unsigned int index, key txnFee);
-    bool verRctMG(mgSig mg, const ctkeyM & pubs, const ctkeyV & outPk, key txnFee);
+    mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const keyV &outMasks, const ctkeyV & outPk, unsigned int index, key txnFee, const key &base_hash);
+    bool verRctMG(mgSig mg, const ctkeyM & pubs, const ctkeyV & outPk, key txnFee, const key &base_hash);
 
     //These functions get keys from blockchain
     //replace these when connecting blockchain
@@ -133,9 +133,10 @@ namespace rct {
     //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1)
     //   uses the attached ecdh info to find the amounts represented by each output commitment
     //   must know the destination private key to find the correct amount, else will return a random number
-    rctSig genRct(const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> amounts, const ctkeyM &mixRing, unsigned int index);
-    rctSig genRct(const ctkeyV & inSk, const ctkeyV  & inPk, const keyV & destinations, const vector<xmr_amount> amounts, const int mixin);
+    rctSig genRct(const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> amounts, const ctkeyM &mixRing, const key &bash_hash, unsigned int index);
+    rctSig genRct(const ctkeyV & inSk, const ctkeyV  & inPk, const keyV & destinations, const vector<xmr_amount> amounts, const key &bash_hash, const int mixin);
     bool verRct(const rctSig & rv);
+    bool verRct(const rctSig & rv, const ctkeyM &mixRing, const keyV &II, const key &base_hash);
     xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask);
     xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i);
 
diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h
index 13e9fb6f2..2dc4718db 100644
--- a/src/ringct/rctTypes.h
+++ b/src/ringct/rctTypes.h
@@ -148,7 +148,16 @@ namespace rct {
         BEGIN_SERIALIZE_OBJECT()
             FIELD(ss)
             FIELD(cc)
-            FIELD(II)
+            if (II.size() == 0) {
+              // loading
+              FIELD(II)
+            }
+            else {
+              // saving
+              keyV II;
+              II.push_back(this->II.back());
+              FIELD(II)
+            }
         END_SERIALIZE()
     };
     //contains the data for an asnl sig
@@ -181,14 +190,16 @@ namespace rct {
         vector<ecdhTuple> ecdhInfo;
         ctkeyV outPk;
         xmr_amount txnFee;
+        key base_hash;
 
         BEGIN_SERIALIZE_OBJECT()
             FIELD(rangeSigs)
             FIELD(MG)
-            FIELD(mixRing)
+            // FIELD(mixRing) - not serialized, it can be reconstructed
             FIELD(ecdhInfo)
             FIELD(outPk)
             FIELD(txnFee)
+            // FIELD(base_hash) - not serialized, it can be reconstructed
         END_SERIALIZE()
     };
     
@@ -296,9 +307,11 @@ namespace rct {
     static inline rct::key pk2rct(const crypto::public_key &pk) { rct::key k; memcpy(&k, &pk, 32); return k; }
     static inline rct::key sk2rct(const crypto::secret_key &sk) { rct::key k; memcpy(&k, &sk, 32); return k; }
     static inline rct::key ki2rct(const crypto::key_image &ki) { rct::key k; memcpy(&k, &ki, 32); return k; }
+    static inline rct::key hash2rct(const crypto::hash &h) { rct::key k; memcpy(&k, &h, 32); return k; }
     static inline crypto::public_key rct2pk(const rct::key &k) { crypto::public_key pk; memcpy(&pk, &k, 32); return pk; }
     static inline crypto::secret_key rct2sk(const rct::key &k) { crypto::secret_key sk; memcpy(&sk, &k, 32); return sk; }
     static inline crypto::key_image rct2ki(const rct::key &k) { crypto::key_image ki; memcpy(&ki, &k, 32); return ki; }
+    static inline crypto::hash rct2hash(const rct::key &k) { crypto::hash h; memcpy(&h, &k, 32); return h; }
     static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !memcmp(&k0, &k1, 32); }
     static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return memcmp(&k0, &k1, 32); }
 }
diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index 59472e648..3565945d4 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -3237,16 +3237,19 @@ static size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs)
 
   // rangeSigs
   size += (2*64*32+32+64*32) * n_outputs;
-  // MG
-  size += 32 * (mixin+1) * n_inputs + 32 + 32 * n_inputs;
-  // mixRing
-  size += 2 * 32 * (mixin+1) * n_inputs;
+
+  // MG - only the last slot of II is saved, the rest can be reconstructed
+  size += 32 * (mixin+1) * n_inputs + 32 + 32 * (/*n_inputs+*/1) ;
+
+  // mixRing - not serialized, can be reconstructed
+  /* size += 2 * 32 * (mixin+1) * n_inputs; */
+
   // ecdhInfo
   size += 3 * 32 * n_outputs;
   // outPk
   size += 2 * 32 * n_outputs;
 
-  LOG_PRINT_L2("estimated rct tx size for " << n_inputs << " at mixin " << mixin << " and " << n_outputs << ": " << size);
+  LOG_PRINT_L2("estimated rct tx size for " << n_inputs << " at mixin " << mixin << " and " << n_outputs << ": " << size << " (" << (32 * n_inputs + 2 * 32 * (mixin+1) * n_inputs) << " saved)");
   return size;
 }