New "Halfway RingCT" outputs for coinbase transactions

When RingCT is enabled, outputs from coinbase transactions
are created as a single output, and stored as RingCT output,
with a fake mask. Their amount is not hidden on the blockchain
itself, but they are then able to be used as fake inputs in
a RingCT ring. Since the output amounts are hidden, their
"dustiness" is not an obstacle anymore to mixing, and this
makes the coinbase transactions a lot smaller, as well as
helping the TXO set to grow more slowly.

Also add a new "Null" type of rct signature, which decreases
the size required when no signatures are to be stored, as
in a coinbase tx.

9 files changed, 92 insertions, 25 deletions

diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp
index 3719d9eb0..7eb81d933 100644
--- a/src/blockchain_db/blockchain_db.cpp
+++ b/src/blockchain_db/blockchain_db.cpp
@@ -46,6 +46,7 @@ void BlockchainDB::pop_block()
 
 void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr)
 {
+  bool miner_tx = false;
   crypto::hash tx_hash;
   if (!tx_hash_ptr)
   {
@@ -67,6 +68,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
     else if (tx_input.type() == typeid(txin_gen))
     {
       /* nothing to do here */
+      miner_tx = true;
     }
     else
     {
@@ -90,8 +92,21 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
   // we need the index
   for (uint64_t i = 0; i < tx.vout.size(); ++i)
   {
-    amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time,
-      tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL));
+    // miner v2 txes have their coinbase output in one single out to save space,
+    // and we store them as rct outputs with an identity mask
+    if (miner_tx && tx.version == 2)
+    {
+      cryptonote::tx_out vout = tx.vout[i];
+      rct::key commitment = rct::zeroCommit(vout.amount);
+      vout.amount = 0;
+      amount_output_indices.push_back(add_output(tx_hash, vout, i, tx.unlock_time,
+        &commitment));
+    }
+    else
+    {
+      amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time,
+        tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL));
+    }
   }
   add_tx_amount_output_indices(tx_id, amount_output_indices);
 }
diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp
index 44dde7759..82691eb6b 100644
--- a/src/cryptonote_core/blockchain.cpp
+++ b/src/cryptonote_core/blockchain.cpp
@@ -959,7 +959,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl
     money_in_use += o.amount;
   partial_block_reward = false;
 
-  if (version >= 3) {
+  if (version == 3) {
     for (auto &o: b.miner_tx.vout) {
       if (!is_valid_decomposed_amount(o.amount)) {
         LOG_PRINT_L1("miner tx output " << print_money(o.amount) << " is not a valid decomposed amount");
@@ -1128,7 +1128,9 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m
    block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size
    */
   //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size
-  bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, 11, m_hardfork->get_current_version());
+  uint8_t hf_version = m_hardfork->get_current_version();
+  size_t max_outs = hf_version >= 4 ? 1 : 11;
+  bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version);
   CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, first chance");
   size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx);
 #if defined(DEBUG_CREATE_BLOCK_TEMPLATE)
@@ -1137,7 +1139,7 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m
 #endif
   for (size_t try_count = 0; try_count != 10; ++try_count)
   {
-    r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, 11, m_hardfork->get_current_version());
+    r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version);
 
     CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, second chance");
     size_t coinbase_blob_size = get_object_blobsize(b.miner_tx);
@@ -2354,17 +2356,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
       }
     }
 
-    // for v3, we force txes with all mixable inputs to be rct
-    if (m_hardfork->get_current_version() >= 4)
-    {
-      if (n_unmixable == 0 && tx.version == 1)
-      {
-        LOG_PRINT_L1("Tx " << get_transaction_hash(tx) << " is not rct and does not have unmixable inputs");
-        tvc.m_not_rct = true;
-        return false;
-      }
-    }
-
     if (mixin < 2)
     {
       if (n_unmixable == 0)
@@ -2543,6 +2534,11 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
     const rct::rctSig &rv = tx.rct_signatures;
     switch (rv.type)
     {
+    case rct::RCTTypeNull: {
+      // we only accept no signatures for coinbase txes
+      LOG_PRINT_L1("Null rct signature on non-coinbase tx");
+      return false;
+    }
     case rct::RCTTypeSimple: {
       // check all this, either recontructed (so should really pass), or not
       {
diff --git a/src/cryptonote_core/cryptonote_basic.h b/src/cryptonote_core/cryptonote_basic.h
index b68040597..f54b8c2b3 100644
--- a/src/cryptonote_core/cryptonote_basic.h
+++ b/src/cryptonote_core/cryptonote_basic.h
@@ -233,6 +233,8 @@ namespace cryptonote
         FIELD(rct_signatures)
         switch (rct_signatures.type)
         {
+        case rct::RCTTypeNull:
+          break;
         case rct::RCTTypeSimple:
           if (rct_signatures.mixRing.size() && rct_signatures.mixRing.size() != vin.size())
             return false;
@@ -276,6 +278,7 @@ namespace cryptonote
     vout.clear();
     extra.clear();
     signatures.clear();
+    rct_signatures.type = rct::RCTTypeNull;
   }
 
   inline
diff --git a/src/cryptonote_core/cryptonote_boost_serialization.h b/src/cryptonote_core/cryptonote_boost_serialization.h
index 09e9a7fa7..c91f78c58 100644
--- a/src/cryptonote_core/cryptonote_boost_serialization.h
+++ b/src/cryptonote_core/cryptonote_boost_serialization.h
@@ -249,6 +249,8 @@ namespace boost
   inline void serialize(Archive &a, rct::rctSigBase &x, const boost::serialization::version_type ver)
   {
     a & x.type;
+    if (x.type == rct::RCTTypeNull)
+      return;
     if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple)
       throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type");
     // a & x.message; message is not serialized, as it can be reconstructed from the tx data
@@ -264,6 +266,8 @@ namespace boost
   inline void serialize(Archive &a, rct::rctSig &x, const boost::serialization::version_type ver)
   {
     a & x.type;
+    if (x.type == rct::RCTTypeNull)
+      return;
     if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple)
       throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type");
     // a & x.message; message is not serialized, as it can be reconstructed from the tx data
diff --git a/src/cryptonote_core/cryptonote_format_utils.cpp b/src/cryptonote_core/cryptonote_format_utils.cpp
index 6a3172d4e..cbaf11398 100644
--- a/src/cryptonote_core/cryptonote_format_utils.cpp
+++ b/src/cryptonote_core/cryptonote_format_utils.cpp
@@ -136,7 +136,10 @@ namespace cryptonote
     // from hard fork 2, we cut out the low significant digits. This makes the tx smaller, and
     // keeps the paid amount almost the same. The unpaid remainder gets pushed back to the
     // emission schedule
-    if (hard_fork_version >= 2) {
+    // from hard fork 4, we use a single "dusty" output. This makes the tx even smaller,
+    // and avoids the quantization. These outputs will be added as rct outputs with identity
+    // masks, to they can be used as rct inputs.
+    if (hard_fork_version >= 2 && hard_fork_version < 4) {
       block_reward = block_reward - block_reward % ::config::BASE_REWARD_CLAMP_THRESHOLD;
     }
 
@@ -146,12 +149,16 @@ namespace cryptonote
       [&out_amounts](uint64_t a_dust) { out_amounts.push_back(a_dust); });
 
     CHECK_AND_ASSERT_MES(1 <= max_outs, false, "max_out must be non-zero");
-    if (height == 0)
+    if (height == 0 || hard_fork_version >= 4)
     {
       // the genesis block was not decomposed, for unknown reasons
       while (max_outs < out_amounts.size())
       {
-        out_amounts[out_amounts.size() - 2] += out_amounts.back();
+        //out_amounts[out_amounts.size() - 2] += out_amounts.back();
+        //out_amounts.resize(out_amounts.size() - 1);
+        out_amounts[1] += out_amounts[0];
+        for (size_t n = 1; n < out_amounts.size(); ++n)
+          out_amounts[n - 1] = out_amounts[n];
         out_amounts.resize(out_amounts.size() - 1);
       }
     }
@@ -182,7 +189,11 @@ namespace cryptonote
 
     CHECK_AND_ASSERT_MES(summary_amounts == block_reward, false, "Failed to construct miner tx, summary_amounts = " << summary_amounts << " not equal block_reward = " << block_reward);
 
-    tx.version = 1;
+    if (hard_fork_version >= 4)
+      tx.version = 2;
+    else
+      tx.version = 1;
+
     //lock
     tx.unlock_time = height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW;
     tx.vin.push_back(in);
@@ -726,7 +737,7 @@ namespace cryptonote
       // zero out all amounts to mask rct outputs, real amounts are now encrypted
       for (size_t i = 0; i < tx.vin.size(); ++i)
       {
-        if (!(sources[i].mask == rct::identity()))
+        if (sources[i].rct)
           boost::get<txin_to_key>(tx.vin[i]).amount = 0;
       }
       for (size_t i = 0; i < tx.vout.size(); ++i)
diff --git a/src/cryptonote_core/cryptonote_format_utils.h b/src/cryptonote_core/cryptonote_format_utils.h
index 5da256921..e6a3bfba4 100644
--- a/src/cryptonote_core/cryptonote_format_utils.h
+++ b/src/cryptonote_core/cryptonote_format_utils.h
@@ -58,6 +58,7 @@ namespace cryptonote
     crypto::public_key real_out_tx_key; //incoming real tx public key
     size_t real_output_in_tx_index;     //index in transaction outputs vector
     uint64_t amount;                    //money
+    bool rct;                           //true if the output is rct
     rct::key mask;                      //ringct amount mask
 
     void push_output(uint64_t idx, const crypto::public_key &k, uint64_t amount) { outputs.push_back(std::make_pair(idx, rct::ctkey({rct::pk2rct(k), rct::zeroCommit(amount)}))); }
diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h
index da2e3808c..58e08b0f7 100644
--- a/src/ringct/rctTypes.h
+++ b/src/ringct/rctTypes.h
@@ -174,8 +174,9 @@ namespace rct {
     // outPk contains public keypairs which are destinations (P, C),
     //  P = address, C = commitment to amount
     enum {
-      RCTTypeFull = 0,
-      RCTTypeSimple = 1,
+      RCTTypeNull = 0,
+      RCTTypeFull = 1,
+      RCTTypeSimple = 2,
     };
     struct rctSigBase {
         uint8_t type;
@@ -189,6 +190,8 @@ namespace rct {
 
         BEGIN_SERIALIZE()
             FIELD(type)
+            if (type == RCTTypeNull)
+              return true;
             // FIELD(message) - not serialized, it can be reconstructed
             // FIELD(mixRing) - not serialized, it can be reconstructed
             if (type == RCTTypeSimple)
@@ -224,6 +227,8 @@ namespace rct {
 
         BEGIN_SERIALIZE_OBJECT()
             FIELDS(*static_cast<rctSigBase *>(this))
+            if (type == RCTTypeNull)
+              return true;
             FIELDS(p);
         END_SERIALIZE()
     };
diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index b3b8e6561..985ebe778 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -489,10 +489,17 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
             {
               td.m_mask = mask[o];
               td.m_amount = amount[o];
+              td.m_rct = true;
+            }
+            else if (miner_tx && tx.version == 2)
+            {
+              td.m_mask = rct::identity();
+              td.m_rct = true;
             }
             else
             {
               td.m_mask = rct::identity();
+              td.m_rct = false;
             }
 	    set_unspent(td);
 	    m_key_images[td.m_key_image] = m_transfers.size()-1;
@@ -529,10 +536,17 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
             {
               td.m_mask = mask[o];
               td.m_amount = amount[o];
+              td.m_rct = true;
+            }
+            else if (miner_tx && tx.version == 2)
+            {
+              td.m_mask = rct::identity();
+              td.m_rct = true;
             }
             else
             {
               td.m_mask = rct::identity();
+              td.m_rct = false;
             }
             THROW_WALLET_EXCEPTION_IF(td.m_key_image != ki[o], error::wallet_internal_error, "Inconsistent key images");
 	    THROW_WALLET_EXCEPTION_IF(td.m_spent, error::wallet_internal_error, "Inconsistent spent status");
@@ -2847,6 +2861,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent
     cryptonote::tx_source_entry& src = sources.back();
     transfer_details& td = *it;
     src.amount = td.amount();
+    src.rct = td.is_rct();
     //paste keys (fake and real)
 
     for (size_t n = 0; n < fake_outputs_count + 1; ++n)
@@ -3037,6 +3052,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry
     cryptonote::tx_source_entry& src = sources.back();
     transfer_details& td = *it;
     src.amount = td.amount();
+    src.rct = td.is_rct();
     //paste mixin transaction
     if(it->is_rct())
     {
@@ -3732,6 +3748,7 @@ void wallet2::transfer_from(const std::vector<size_t> &outs, size_t num_outputs,
     cryptonote::tx_source_entry& src = sources.back();
     transfer_details& td = *it;
     src.amount = td.amount();
+    src.rct = td.is_rct();
 
     //paste real transaction to the random index
     auto it_to_insert = std::find_if(src.outputs.begin(), src.outputs.end(), [&](const tx_output_entry& a)
diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h
index 0d61f90e2..259f7aca7 100644
--- a/src/wallet/wallet2.h
+++ b/src/wallet/wallet2.h
@@ -108,8 +108,9 @@ namespace tools
       crypto::key_image m_key_image; //TODO: key_image stored twice :(
       rct::key m_mask;
       uint64_t m_amount;
+      bool m_rct;
 
-      bool is_rct() const { return m_tx.vout[m_internal_output_index].amount == 0; }
+      bool is_rct() const { return m_rct; }
       uint64_t amount() const { return m_amount; }
     };
 
@@ -502,7 +503,7 @@ namespace tools
   };
 }
 BOOST_CLASS_VERSION(tools::wallet2, 14)
-BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 3)
+BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 4)
 BOOST_CLASS_VERSION(tools::wallet2::payment_details, 1)
 BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 5)
 BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 2)
@@ -527,6 +528,10 @@ namespace boost
         {
           x.m_spent_height = 0;
         }
+        if (ver < 4)
+        {
+          x.m_rct = x.m_tx.vout[x.m_internal_output_index].amount == 0;
+        }
     }
 
     template <class Archive>
@@ -563,8 +568,17 @@ namespace boost
       }
       a & x.m_spent_height;
       if (ver < 3)
+      {
+        initialize_transfer_details(a, x, ver);
         return;
+      }
       a & x.m_txid;
+      if (ver < 4)
+      {
+        initialize_transfer_details(a, x, ver);
+        return;
+      }
+      a & x.m_rct;
     }
 
     template <class Archive>
@@ -770,6 +784,7 @@ namespace tools
       cryptonote::tx_source_entry& src = sources.back();
       transfer_details& td = *it;
       src.amount = td.amount();
+      src.rct = false;
       //paste mixin transaction
       if(daemon_resp.outs.size())
       {