21 files changed, 338 insertions, 99 deletions
diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp
index ad3febbd1..a990d7aaf 100644
--- a/src/blockchain_db/berkeleydb/db_bdb.cpp
+++ b/src/blockchain_db/berkeleydb/db_bdb.cpp
@@ -1264,19 +1264,6 @@ uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const
     return (uint64_t)result - 1;
 }
 
-//FIXME: make sure the random method used here is appropriate
-uint64_t BlockchainBDB::get_random_output(const uint64_t& amount) const
-{
-    LOG_PRINT_L3("BlockchainBDB::" << __func__);
-    check_open();
-
-    uint64_t num_outputs = get_num_outputs(amount);
-    if (num_outputs == 0)
-        throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist"));
-
-    return crypto::rand<uint64_t>() % num_outputs;
-}
-
 uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const
 {
     LOG_PRINT_L3("BlockchainBDB::" << __func__);
diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h
index 41f4bcb78..f92bbef68 100644
--- a/src/blockchain_db/berkeleydb/db_bdb.h
+++ b/src/blockchain_db/berkeleydb/db_bdb.h
@@ -295,8 +295,6 @@ public:
 
   virtual uint64_t get_tx_block_height(const crypto::hash& h) const;
 
-  virtual uint64_t get_random_output(const uint64_t& amount) const;
-
   virtual uint64_t get_num_outputs(const uint64_t& amount) const;
 
   virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index);
diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h
index ff15109b0..25a34fc09 100644
--- a/src/blockchain_db/blockchain_db.h
+++ b/src/blockchain_db/blockchain_db.h
@@ -104,7 +104,6 @@
  *   height      get_tx_block_height(hash)
  *
  * Outputs:
- *   index       get_random_output(amount)
  *   uint64_t    get_num_outputs(amount)
  *   pub_key     get_output_key(amount, index)
  *   tx_out      get_output(tx_hash, index)
@@ -463,9 +462,6 @@ public:
   // returns height of block that contains transaction with hash <h>
   virtual uint64_t get_tx_block_height(const crypto::hash& h) const = 0;
 
-  // return global output index of a random output of amount <amount>
-  virtual uint64_t get_random_output(const uint64_t& amount) const = 0;
-
   // returns the total number of outputs of amount <amount>
   virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0;
 
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index dd829f3b0..1583a0c06 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -28,6 +28,7 @@
 #include "db_lmdb.h"
 
 #include <boost/filesystem.hpp>
+#include <boost/format.hpp>
 #include <memory>  // std::unique_ptr
 #include <cstring>  // memcpy
 #include <random>
@@ -262,6 +263,7 @@ void mdb_txn_safe::allow_new_txns()
 
 void BlockchainLMDB::do_resize(uint64_t increase_size)
 {
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   CRITICAL_REGION_LOCAL(m_synchronization_lock);
   const uint64_t add_size = 1LL << 30;
 
@@ -327,6 +329,7 @@ void BlockchainLMDB::do_resize(uint64_t increase_size)
 // threshold_size is used for batch transactions
 bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
 {
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
 #if defined(ENABLE_AUTO_RESIZE)
   MDB_envinfo mei;
 
@@ -346,7 +349,8 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
   LOG_PRINT_L1("Space used:      " << size_used);
   LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used);
   LOG_PRINT_L1("Size threshold:  " << threshold_size);
-  LOG_PRINT_L1("Percent used: " << (double)size_used/mei.me_mapsize << "  Percent threshold: " << RESIZE_PERCENT);
+  float resize_percent_old = RESIZE_PERCENT;
+  LOG_PRINT_L1(boost::format("Percent used: %.04f  Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent_old);
 
   if (threshold_size > 0)
   {
@@ -376,7 +380,8 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
 
 void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
 {
-  LOG_PRINT_L1("[batch] checking DB size");
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size");
   const uint64_t min_increase_size = 128 * (1 << 20);
   uint64_t threshold_size = 0;
   uint64_t increase_size = 0;
@@ -407,6 +412,7 @@ void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
 
 uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) const
 {
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   uint64_t threshold_size = 0;
 
   // batch size estimate * batch safety factor = final size estimate
@@ -419,22 +425,33 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
   // For resizing purposes, allow for at least 4k average block size.
   uint64_t min_block_size = 4 * 1024;
 
-  uint64_t block_stop = m_height - 1;
+  uint64_t block_stop = 0;
+  if (m_height > 1)
+    block_stop = m_height - 1;
   uint64_t block_start = 0;
   if (block_stop >= num_prev_blocks)
     block_start = block_stop - num_prev_blocks + 1;
   uint32_t num_blocks_used = 0;
   uint64_t total_block_size = 0;
-  for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
+  LOG_PRINT_L1("[" << __func__ << "] " << "m_height: " << m_height << "  block_start: " << block_start << "  block_stop: " << block_stop);
+  size_t avg_block_size = 0;
+  if (m_height == 0)
   {
-    uint32_t block_size = get_block_size(block_num);
-    total_block_size += block_size;
-    // Track number of blocks being totalled here instead of assuming, in case
-    // some blocks were to be skipped for being outliers.
-    ++num_blocks_used;
+    LOG_PRINT_L1("No existing blocks to check for average block size");
+  }
+  else
+  {
+    for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
+    {
+      uint32_t block_size = get_block_size(block_num);
+      total_block_size += block_size;
+      // Track number of blocks being totalled here instead of assuming, in case
+      // some blocks were to be skipped for being outliers.
+      ++num_blocks_used;
+    }
+    avg_block_size = total_block_size / num_blocks_used;
+    LOG_PRINT_L1("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size);
   }
-  size_t avg_block_size = total_block_size / num_blocks_used;
-  LOG_PRINT_L1("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size);
   if (avg_block_size < min_block_size)
     avg_block_size = min_block_size;
   LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size);
@@ -1620,19 +1637,6 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
   return *(const uint64_t*)result.mv_data;
 }
 
-//FIXME: make sure the random method used here is appropriate
-uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) const
-{
-  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
-  check_open();
-
-  uint64_t num_outputs = get_num_outputs(amount);
-  if (num_outputs == 0)
-    throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist"));
-
-  return crypto::rand<uint64_t>() % num_outputs;
-}
-
 uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index 0facb8713..b4c197803 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -157,8 +157,6 @@ public:
 
   virtual uint64_t get_tx_block_height(const crypto::hash& h) const;
 
-  virtual uint64_t get_random_output(const uint64_t& amount) const;
-
   virtual uint64_t get_num_outputs(const uint64_t& amount) const;
 
   virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index);
diff --git a/src/crypto/hash.h b/src/crypto/hash.h
index 2f91a5358..1ce2f9816 100644
--- a/src/crypto/hash.h
+++ b/src/crypto/hash.h
@@ -45,9 +45,13 @@ namespace crypto {
   POD_CLASS hash {
     char data[HASH_SIZE];
   };
+  POD_CLASS hash8 {
+    char data[8];
+  };
 #pragma pack(pop)
 
   static_assert(sizeof(hash) == HASH_SIZE, "Invalid structure size");
+  static_assert(sizeof(hash8) == 8, "Invalid structure size");
 
   /*
     Cryptonight hash functions
@@ -74,3 +78,4 @@ namespace crypto {
 }
 
 CRYPTO_MAKE_HASHABLE(hash)
+CRYPTO_MAKE_COMPARABLE(hash8)
diff --git a/src/cryptonote_core/account.cpp b/src/cryptonote_core/account.cpp
index eb79f5949..fe8dbd1e8 100644
--- a/src/cryptonote_core/account.cpp
+++ b/src/cryptonote_core/account.cpp
@@ -120,7 +120,7 @@ DISABLE_VS_WARNINGS(4244 4345)
     return get_account_address_as_str(testnet, m_keys.m_account_address);
   }
   //-----------------------------------------------------------------
-  std::string account_base::get_public_integrated_address_str(const crypto::hash &payment_id, bool testnet) const
+  std::string account_base::get_public_integrated_address_str(const crypto::hash8 &payment_id, bool testnet) const
   {
     //TODO: change this code into base 58
     return get_account_integrated_address_as_str(testnet, m_keys.m_account_address, payment_id);
diff --git a/src/cryptonote_core/account.h b/src/cryptonote_core/account.h
index 088363bf1..732645ee4 100644
--- a/src/cryptonote_core/account.h
+++ b/src/cryptonote_core/account.h
@@ -61,7 +61,7 @@ namespace cryptonote
     void create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey);
     const account_keys& get_keys() const;
     std::string get_public_address_str(bool testnet) const;
-    std::string get_public_integrated_address_str(const crypto::hash &payment_id, bool testnet) const;
+    std::string get_public_integrated_address_str(const crypto::hash8 &payment_id, bool testnet) const;
 
     uint64_t get_createtime() const { return m_creation_timestamp; }
     void set_createtime(uint64_t val) { m_creation_timestamp = val; }
diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp
index cc2d6f821..648b77b2f 100644
--- a/src/cryptonote_core/blockchain.cpp
+++ b/src/cryptonote_core/blockchain.cpp
@@ -1473,6 +1473,7 @@ bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUT
     // from BlockchainDB where <n> is req.outs_count (number of mixins).
     for (uint64_t amount : req.amounts)
     {
+        auto num_outs = m_db->get_num_outputs(amount);
         // create outs_for_amount struct and populate amount field
         COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount());
         result_outs.amount = amount;
@@ -1481,9 +1482,9 @@ bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUT
 
         // if there aren't enough outputs to mix with (or just enough),
         // use all of them.  Eventually this should become impossible.
-        if (m_db->get_num_outputs(amount) <= req.outs_count)
+        if (num_outs <= req.outs_count)
         {
-            for (uint64_t i = 0; i < m_db->get_num_outputs(amount); i++)
+            for (uint64_t i = 0; i < num_outs; i++)
             {
                 // get tx_hash, tx_out_index from DB
                 tx_out_index toi = m_db->get_output_tx_and_index(amount, i);
@@ -1499,7 +1500,6 @@ bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUT
         else
         {
             // while we still need more mixins
-            auto num_outs = m_db->get_num_outputs(amount);
             while (result_outs.outs.size() < req.outs_count)
             {
                 // if we've gone through every possible output, we've gotten all we can
@@ -1511,7 +1511,15 @@ bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUT
                 // get a random output index from the DB.  If we've already seen it,
                 // return to the top of the loop and try again, otherwise add it to the
                 // list of output indices we've seen.
-                uint64_t i = m_db->get_random_output(amount);
+
+                // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit
+                uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53);
+                double frac = std::sqrt((double)r / ((uint64_t)1 << 53));
+                uint64_t i = (uint64_t)(frac*num_outs);
+                // just in case rounding up to 1 occurs after sqrt
+                if (i == num_outs)
+                  --i;
+
                 if (seen_indices.count(i))
                 {
                     continue;
diff --git a/src/cryptonote_core/cryptonote_basic.h b/src/cryptonote_core/cryptonote_basic.h
index 2be76c0de..07745bf0d 100644
--- a/src/cryptonote_core/cryptonote_basic.h
+++ b/src/cryptonote_core/cryptonote_basic.h
@@ -57,6 +57,7 @@ namespace cryptonote
 {
 
   const static crypto::hash null_hash = AUTO_VAL_INIT(null_hash);
+  const static crypto::hash8 null_hash8 = AUTO_VAL_INIT(null_hash8);
   const static crypto::public_key null_pkey = AUTO_VAL_INIT(null_pkey);
 
   typedef std::vector<crypto::signature> ring_signature;
diff --git a/src/cryptonote_core/cryptonote_basic_impl.cpp b/src/cryptonote_core/cryptonote_basic_impl.cpp
index 1319a2ef9..bd0b8a304 100644
--- a/src/cryptonote_core/cryptonote_basic_impl.cpp
+++ b/src/cryptonote_core/cryptonote_basic_impl.cpp
@@ -46,7 +46,7 @@ namespace cryptonote {
 
   struct integrated_address {
     account_public_address adr;
-    crypto::hash payment_id;
+    crypto::hash8 payment_id;
 
     BEGIN_SERIALIZE_OBJECT()
       FIELD(adr)
@@ -150,7 +150,7 @@ namespace cryptonote {
   std::string get_account_integrated_address_as_str(
       bool testnet
     , account_public_address const & adr
-    , crypto::hash const & payment_id
+    , crypto::hash8 const & payment_id
     )
   {
     uint64_t integrated_address_prefix = testnet ?
@@ -176,7 +176,7 @@ namespace cryptonote {
   bool get_account_integrated_address_from_str(
       account_public_address& adr
     , bool& has_payment_id
-    , crypto::hash& payment_id
+    , crypto::hash8& payment_id
     , bool testnet
     , std::string const & str
     )
@@ -278,7 +278,7 @@ namespace cryptonote {
     )
   {
     bool has_payment_id;
-    crypto::hash payment_id;
+    crypto::hash8 payment_id;
     return get_account_integrated_address_from_str(adr, has_payment_id, payment_id, testnet, str);
   }
 
diff --git a/src/cryptonote_core/cryptonote_basic_impl.h b/src/cryptonote_core/cryptonote_basic_impl.h
index 87d6f1024..5c442d558 100644
--- a/src/cryptonote_core/cryptonote_basic_impl.h
+++ b/src/cryptonote_core/cryptonote_basic_impl.h
@@ -60,7 +60,7 @@ namespace cryptonote {
   {
     uint8_t m_ver;
     account_public_address m_address;
-    crypto::hash payment_id;
+    crypto::hash8 payment_id;
     uint8_t check_sum;
   };
 #pragma pack (pop)
@@ -83,13 +83,13 @@ namespace cryptonote {
   std::string get_account_integrated_address_as_str(
       bool testnet
     , const account_public_address& adr
-    , const crypto::hash& payment_id
+    , const crypto::hash8& payment_id
     );
 
   bool get_account_integrated_address_from_str(
       account_public_address& adr
     , bool& has_payment_id
-    , crypto::hash& payment_id
+    , crypto::hash8& payment_id
     , bool testnet
     , const std::string& str
     );
@@ -110,6 +110,10 @@ template <class T>
 std::ostream &print256(std::ostream &o, const T &v) {
   return o << "<" << epee::string_tools::pod_to_hex(v) << ">";
 }
+template <class T>
+std::ostream &print64(std::ostream &o, const T &v) {
+  return o << "<" << epee::string_tools::pod_to_hex(v) << ">";
+}
 
 bool parse_hash256(const std::string str_hash, crypto::hash& hash);
 
@@ -120,4 +124,5 @@ namespace crypto {
   inline std::ostream &operator <<(std::ostream &o, const crypto::key_image &v) { return print256(o, v); }
   inline std::ostream &operator <<(std::ostream &o, const crypto::signature &v) { return print256(o, v); }
   inline std::ostream &operator <<(std::ostream &o, const crypto::hash &v) { return print256(o, v); }
+  inline std::ostream &operator <<(std::ostream &o, const crypto::hash8 &v) { return print64(o, v); }
 }
diff --git a/src/cryptonote_core/cryptonote_format_utils.cpp b/src/cryptonote_core/cryptonote_format_utils.cpp
index 1b21894a0..af8e703bf 100644
--- a/src/cryptonote_core/cryptonote_format_utils.cpp
+++ b/src/cryptonote_core/cryptonote_format_utils.cpp
@@ -38,6 +38,8 @@ using namespace epee;
 #include "crypto/crypto.h"
 #include "crypto/hash.h"
 
+#define ENCRYPTED_PAYMENT_ID_TAIL 0x8d
+
 namespace cryptonote
 {
   //---------------------------------------------------------------
@@ -303,6 +305,35 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
+  bool remove_extra_nonce_tx_extra(std::vector<uint8_t>& tx_extra)
+  {
+    std::string extra_str(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size());
+    std::istringstream iss(extra_str);
+    binary_archive<false> ar(iss);
+    std::ostringstream oss;
+    binary_archive<true> newar(oss);
+
+    bool eof = false;
+    while (!eof)
+    {
+      tx_extra_field field;
+      bool r = ::do_serialize(ar, field);
+      CHECK_AND_NO_ASSERT_MES(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
+      if (field.type() != typeid(tx_extra_nonce))
+        ::do_serialize(newar, field);
+
+      std::ios_base::iostate state = iss.rdstate();
+      eof = (EOF == iss.peek());
+      iss.clear(state);
+    }
+    CHECK_AND_NO_ASSERT_MES(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size())));
+    tx_extra.clear();
+    std::string s = oss.str();
+    tx_extra.reserve(s.size());
+    std::copy(s.begin(), s.end(), std::back_inserter(tx_extra));
+    return true;
+  }
+  //---------------------------------------------------------------
   void set_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash& payment_id)
   {
     extra_nonce.clear();
@@ -311,6 +342,14 @@ namespace cryptonote
     std::copy(payment_id_ptr, payment_id_ptr + sizeof(payment_id), std::back_inserter(extra_nonce));
   }
   //---------------------------------------------------------------
+  void set_encrypted_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash8& payment_id)
+  {
+    extra_nonce.clear();
+    extra_nonce.push_back(TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID);
+    const uint8_t* payment_id_ptr = reinterpret_cast<const uint8_t*>(&payment_id);
+    std::copy(payment_id_ptr, payment_id_ptr + sizeof(payment_id), std::back_inserter(extra_nonce));
+  }
+  //---------------------------------------------------------------
   bool get_payment_id_from_tx_extra_nonce(const blobdata& extra_nonce, crypto::hash& payment_id)
   {
     if(sizeof(crypto::hash) + 1 != extra_nonce.size())
@@ -321,6 +360,54 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
+  bool get_encrypted_payment_id_from_tx_extra_nonce(const blobdata& extra_nonce, crypto::hash8& payment_id)
+  {
+    if(sizeof(crypto::hash8) + 1 != extra_nonce.size())
+      return false;
+    if (TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID != extra_nonce[0])
+      return false;
+    payment_id = *reinterpret_cast<const crypto::hash8*>(extra_nonce.data() + 1);
+    return true;
+  }
+  //---------------------------------------------------------------
+  crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const account_keys &sender_keys)
+  {
+    if (destinations.empty())
+      return null_pkey;
+    for (size_t n = 1; n < destinations.size(); ++n)
+    {
+      if (!memcmp(&destinations[n].addr, &sender_keys.m_account_address, sizeof(destinations[0].addr)))
+        continue;
+      if (memcmp(&destinations[n].addr, &destinations[0].addr, sizeof(destinations[0].addr)))
+        return null_pkey;
+    }
+    return destinations[0].addr.m_view_public_key;
+  }
+  //---------------------------------------------------------------
+  bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key)
+  {
+    crypto::key_derivation derivation;
+    crypto::hash hash;
+    char data[33]; /* A hash, and an extra byte */
+
+    if (!generate_key_derivation(public_key, secret_key, derivation))
+      return false;
+
+    memcpy(data, &derivation, 32);
+    data[32] = ENCRYPTED_PAYMENT_ID_TAIL;
+    cn_fast_hash(data, 33, hash);
+
+    for (size_t b = 0; b < 8; ++b)
+      payment_id.data[b] ^= hash.data[b];
+
+    return true;
+  }
+  bool decrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key)
+  {
+    // Encryption and decryption are the same operation (xor with a key)
+    return encrypt_payment_id(payment_id, public_key, secret_key);
+  }
+  //---------------------------------------------------------------
   bool construct_tx(const account_keys& sender_account_keys, const std::vector<tx_source_entry>& sources, const std::vector<tx_destination_entry>& destinations, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time)
   {
     tx.vin.clear();
@@ -334,6 +421,49 @@ namespace cryptonote
     keypair txkey = keypair::generate();
     add_tx_pub_key_to_extra(tx, txkey.pub);
 
+    // if we have a stealth payment id, find it and encrypt it with the tx key now
+    std::vector<tx_extra_field> tx_extra_fields;
+    if (parse_tx_extra(tx.extra, tx_extra_fields))
+    {
+      tx_extra_nonce extra_nonce;
+      if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce))
+      {
+        crypto::hash8 payment_id = null_hash8;
+        if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id))
+        {
+          LOG_PRINT_L2("Encrypting payment id " << payment_id);
+          crypto::key_derivation derivation;
+          crypto::public_key view_key_pub = get_destination_view_key_pub(destinations, sender_account_keys);
+          if (view_key_pub == null_pkey)
+          {
+            LOG_ERROR("Destinations have to have exactly one output to support encrypted payment ids");
+            return false;
+          }
+
+          if (!encrypt_payment_id(payment_id, view_key_pub, txkey.sec))
+          {
+            LOG_ERROR("Failed to encrypt payment id");
+            return false;
+          }
+
+          std::string extra_nonce;
+          set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id);
+          remove_extra_nonce_tx_extra(tx.extra);
+          if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce))
+          {
+            LOG_ERROR("Failed to add encrypted payment id to tx extra");
+            return false;
+          }
+          LOG_PRINT_L1("Encrypted payment ID: " << payment_id);
+        }
+      }
+    }
+    else
+    {
+      LOG_ERROR("Failed to parse tx extra");
+      return false;
+    }
+
     struct input_generation_context_data
     {
       keypair in_ephemeral;
diff --git a/src/cryptonote_core/cryptonote_format_utils.h b/src/cryptonote_core/cryptonote_format_utils.h
index fd785aafd..319205368 100644
--- a/src/cryptonote_core/cryptonote_format_utils.h
+++ b/src/cryptonote_core/cryptonote_format_utils.h
@@ -45,6 +45,8 @@ namespace cryptonote
   bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash);
   bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx);
   bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 1);
+  bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key);
+  bool decrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key);
 
   struct tx_source_entry
   {
@@ -85,8 +87,11 @@ namespace cryptonote
   crypto::public_key get_tx_pub_key_from_extra(const transaction& tx);
   bool add_tx_pub_key_to_extra(transaction& tx, const crypto::public_key& tx_pub_key);
   bool add_extra_nonce_to_tx_extra(std::vector<uint8_t>& tx_extra, const blobdata& extra_nonce);
+  bool remove_extra_nonce_tx_extra(std::vector<uint8_t>& tx_extra);
   void set_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash& payment_id);
+  void set_encrypted_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash8& payment_id);
   bool get_payment_id_from_tx_extra_nonce(const blobdata& extra_nonce, crypto::hash& payment_id);
+  bool get_encrypted_payment_id_from_tx_extra_nonce(const blobdata& extra_nonce, crypto::hash8& payment_id);
   bool is_out_to_acc(const account_keys& acc, const txout_to_key& out_key, const crypto::public_key& tx_pub_key, size_t output_index);
   bool lookup_acc_outs(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector<size_t>& outs, uint64_t& money_transfered);
   bool lookup_acc_outs(const account_keys& acc, const transaction& tx, std::vector<size_t>& outs, uint64_t& money_transfered);
diff --git a/src/cryptonote_core/tx_extra.h b/src/cryptonote_core/tx_extra.h
index ccfe4d1d9..012f41593 100644
--- a/src/cryptonote_core/tx_extra.h
+++ b/src/cryptonote_core/tx_extra.h
@@ -40,6 +40,7 @@
 #define TX_EXTRA_MERGE_MINING_TAG           0x03
 
 #define TX_EXTRA_NONCE_PAYMENT_ID           0x00
+#define TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID 0x01
 
 namespace cryptonote
 {
diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp
index 4ba9a5f32..2f3a6b4d5 100644
--- a/src/daemon/rpc_command_executor.cpp
+++ b/src/daemon/rpc_command_executor.cpp
@@ -482,6 +482,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash) {
   }
   else
   {
+    req.txs_hashes.push_back(epee::string_tools::pod_to_hex(transaction_hash));
     if (!m_rpc_server->on_get_transactions(req, res))
     {
       tools::fail_msg_writer() << fail_message.c_str();
@@ -491,11 +492,29 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash) {
 
   if (1 == res.txs_as_hex.size())
   {
+    // first as hex
     tools::success_msg_writer() << res.txs_as_hex.front();
+
+    // then as json
+    crypto::hash tx_hash, tx_prefix_hash;
+    cryptonote::transaction tx;
+    cryptonote::blobdata blob;
+    if (!string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), blob))
+    {
+      tools::fail_msg_writer() << "Failed to parse tx";
+    }
+    else if (!cryptonote::parse_and_validate_tx_from_blob(blob, tx, tx_hash, tx_prefix_hash))
+    {
+      tools::fail_msg_writer() << "Failed to parse tx blob";
+    }
+    else
+    {
+      tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl;
+    }
   }
   else
   {
-    tools::fail_msg_writer() << "transaction wasn't found: <" << transaction_hash << '>' << std::endl;
+    tools::fail_msg_writer() << "transaction wasn't found: " << transaction_hash << std::endl;
   }
 
   return true;
diff --git a/src/serialization/crypto.h b/src/serialization/crypto.h
index f18e85b12..575697c06 100644
--- a/src/serialization/crypto.h
+++ b/src/serialization/crypto.h
@@ -79,12 +79,14 @@ bool do_serialize(Archive<true> &ar, std::vector<crypto::signature> &v)
 
 BLOB_SERIALIZER(crypto::chacha8_iv);
 BLOB_SERIALIZER(crypto::hash);
+BLOB_SERIALIZER(crypto::hash8);
 BLOB_SERIALIZER(crypto::public_key);
 BLOB_SERIALIZER(crypto::secret_key);
 BLOB_SERIALIZER(crypto::key_derivation);
 BLOB_SERIALIZER(crypto::key_image);
 BLOB_SERIALIZER(crypto::signature);
 VARIANT_TAG(debug_archive, crypto::hash, "hash");
+VARIANT_TAG(debug_archive, crypto::hash8, "hash8");
 VARIANT_TAG(debug_archive, crypto::public_key, "public_key");
 VARIANT_TAG(debug_archive, crypto::secret_key, "secret_key");
 VARIANT_TAG(debug_archive, crypto::key_derivation, "key_derivation");
diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp
index cad83bb45..01d052ce4 100644
--- a/src/simplewallet/simplewallet.cpp
+++ b/src/simplewallet/simplewallet.cpp
@@ -601,7 +601,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
       // parse address
       cryptonote::account_public_address address;
       bool has_payment_id;
-      crypto::hash new_payment_id;
+      crypto::hash8 new_payment_id;
       if(!get_account_integrated_address_from_str(address, has_payment_id, new_payment_id, testnet, parts[0]))
       {
           fail_msg_writer() << tr("Failed to parse address");
@@ -1294,29 +1294,39 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str
     local_args.pop_back();
 
     crypto::hash payment_id;
-    bool r = tools::wallet2::parse_payment_id(payment_id_str, payment_id);
+    bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id);
     if(r)
     {
       std::string extra_nonce;
       set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id);
       r = add_extra_nonce_to_tx_extra(extra, extra_nonce);
     }
+    else
+    {
+      crypto::hash8 payment_id8;
+      r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8);
+      if(r)
+      {
+        std::string extra_nonce;
+        set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8);
+        r = add_extra_nonce_to_tx_extra(extra, extra_nonce);
+      }
+    }
 
     if(!r)
     {
-      fail_msg_writer() << tr("payment id has invalid format, expected 64-character string: ") << payment_id_str;
+      fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character string: ") << payment_id_str;
       return true;
     }
     payment_id_seen = true;
   }
 
   vector<cryptonote::tx_destination_entry> dsts;
-  crypto::hash payment_id = null_hash;
   for (size_t i = 0; i < local_args.size(); i += 2)
   {
     cryptonote::tx_destination_entry de;
     bool has_payment_id;
-    crypto::hash new_payment_id;
+    crypto::hash8 new_payment_id;
     if(!get_account_integrated_address_from_str(de.addr, has_payment_id, new_payment_id, m_wallet->testnet(), local_args[i]))
     {
       // if treating as an address fails, try as url
@@ -1377,23 +1387,21 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str
       }
     }
 
-    if (has_payment_id) {
-      if (payment_id_seen && payment_id != new_payment_id) {
+    if (has_payment_id)
+    {
+      if (payment_id_seen)
+      {
         fail_msg_writer() << tr("A single transaction cannot use more than one payment id: ") << local_args[i];
         return true;
       }
 
-      if (!payment_id_seen)
+      std::string extra_nonce;
+      set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, new_payment_id);
+      bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce);
+      if(!r)
       {
-        std::string extra_nonce;
-        set_payment_id_to_tx_extra_nonce(extra_nonce, new_payment_id);
-        bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce);
-        if(!r)
-        {
-          fail_msg_writer() << tr("Failed to set up payment id, though it was decoded correctly");
-          return true;
-        }
-        payment_id = new_payment_id;
+        fail_msg_writer() << tr("Failed to set up payment id, though it was decoded correctly");
+        return true;
       }
     }
 
@@ -1696,7 +1704,7 @@ bool simple_wallet::print_address(const std::vector<std::string> &args/* = std::
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::print_integrated_address(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
-  crypto::hash payment_id;
+  crypto::hash8 payment_id;
   if (args.size() > 1)
   {
     fail_msg_writer() << tr("integrated_address only takes one or zero arguments");
@@ -1704,19 +1712,19 @@ bool simple_wallet::print_integrated_address(const std::vector<std::string> &arg
   }
   if (args.size() == 0)
   {
-    crypto::generate_random_bytes(32, payment_id.data);
+    crypto::generate_random_bytes(8, payment_id.data);
     success_msg_writer() << tr("Random payment ID: ") << payment_id;
     success_msg_writer() << tr("Matching integrated address: ") << m_wallet->get_account().get_public_integrated_address_str(payment_id, m_wallet->testnet());
     return true;
   }
-  if(tools::wallet2::parse_payment_id(args.back(), payment_id))
+  if(tools::wallet2::parse_short_payment_id(args.back(), payment_id))
   {
     success_msg_writer() << m_wallet->get_account().get_public_integrated_address_str(payment_id, m_wallet->testnet());
     return true;
   }
   else {
     bool has_payment_id;
-    crypto::hash payment_id;
+    crypto::hash8 payment_id;
     account_public_address addr;
     if(get_account_integrated_address_from_str(addr, has_payment_id, payment_id, m_wallet->testnet(), args.back()))
     {
diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index 5b7b19520..3d3f1e4de 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -150,6 +150,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_
   process_unconfirmed(tx);
   std::vector<size_t> outs;
   uint64_t tx_money_got_in_outs = 0;
+  crypto::public_key tx_pub_key = null_pkey;
 
   std::vector<tx_extra_field> tx_extra_fields;
   if(!parse_tx_extra(tx.extra, tx_extra_fields))
@@ -170,7 +171,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_
       return;
     }
 
-    crypto::public_key tx_pub_key = pub_key_field.pub_key;
+    tx_pub_key = pub_key_field.pub_key;
     bool r = lookup_acc_outs(m_account.get_keys(), tx, tx_pub_key, outs, tx_money_got_in_outs);
     THROW_WALLET_EXCEPTION_IF(!r, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys());
 
@@ -236,9 +237,34 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_
   crypto::hash payment_id = null_hash;
   if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce))
   {
-    if(get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id))
+    crypto::hash8 payment_id8 = null_hash8;
+    if(get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8))
     {
       // We got a payment ID to go with this tx
+      LOG_PRINT_L2("Found encrypted payment ID: " << payment_id8);
+      if (tx_pub_key != null_pkey)
+      {
+        if (!decrypt_payment_id(payment_id8, tx_pub_key, m_account.get_keys().m_view_secret_key))
+        {
+          LOG_PRINT_L0("Failed to decrypt payment ID: " << payment_id8);
+        }
+        else
+        {
+          LOG_PRINT_L2("Decrypted payment ID: " << payment_id8);
+          // put the 64 bit decrypted payment id in the first 8 bytes
+          memcpy(payment_id.data, payment_id8.data, 8);
+          // rest is already 0, but guard against code changes above
+          memset(payment_id.data + 8, 0, 24);
+        }
+      }
+      else
+      {
+        LOG_PRINT_L1("No public key found in tx, unable to decrypt payment id");
+      }
+    }
+    else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id))
+    {
+      LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id);
     }
   }
   uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0;
@@ -765,7 +791,7 @@ bool wallet2::wallet_valid_path_format(const std::string& file_path)
   return !file_path.empty();
 }
 //----------------------------------------------------------------------------------------------------
-bool wallet2::parse_payment_id(const std::string& payment_id_str, crypto::hash& payment_id)
+bool wallet2::parse_long_payment_id(const std::string& payment_id_str, crypto::hash& payment_id)
 {
   cryptonote::blobdata payment_id_data;
   if(!epee::string_tools::parse_hexstr_to_binbuff(payment_id_str, payment_id_data))
@@ -778,6 +804,33 @@ bool wallet2::parse_payment_id(const std::string& payment_id_str, crypto::hash&
   return true;
 }
 //----------------------------------------------------------------------------------------------------
+bool wallet2::parse_short_payment_id(const std::string& payment_id_str, crypto::hash8& payment_id)
+{
+  cryptonote::blobdata payment_id_data;
+  if(!epee::string_tools::parse_hexstr_to_binbuff(payment_id_str, payment_id_data))
+    return false;
+
+  if(sizeof(crypto::hash8) != payment_id_data.size())
+    return false;
+
+  payment_id = *reinterpret_cast<const crypto::hash8*>(payment_id_data.data());
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
+bool wallet2::parse_payment_id(const std::string& payment_id_str, crypto::hash& payment_id)
+{
+  if (parse_long_payment_id(payment_id_str, payment_id))
+    return true;
+  crypto::hash8 payment_id8;
+  if (parse_short_payment_id(payment_id_str, payment_id8))
+  {
+    memcpy(payment_id.data, payment_id8.data, 8);
+    memset(payment_id.data + 8, 0, 24);
+    return true;
+  }
+  return false;
+}
+//----------------------------------------------------------------------------------------------------
 bool wallet2::prepare_file_names(const std::string& file_path)
 {
   do_prepare_file_names(file_path, m_keys_file, m_wallet_file);
diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h
index 93119cd0b..eee6aa58c 100644
--- a/src/wallet/wallet2.h
+++ b/src/wallet/wallet2.h
@@ -266,6 +266,8 @@ namespace tools
      */
     static bool wallet_valid_path_format(const std::string& file_path);
 
+    static bool parse_long_payment_id(const std::string& payment_id_str, crypto::hash& payment_id);
+    static bool parse_short_payment_id(const std::string& payment_id_str, crypto::hash8& payment_id);
     static bool parse_payment_id(const std::string& payment_id_str, crypto::hash& payment_id);
 
     static std::vector<std::string> addresses_from_url(const std::string& url, bool& dnssec_valid);
diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp
index 1d38695ae..7dfd64eef 100644
--- a/src/wallet/wallet_rpc_server.cpp
+++ b/src/wallet/wallet_rpc_server.cpp
@@ -119,12 +119,13 @@ namespace tools
   //------------------------------------------------------------------------------------------------------------------------------
   bool wallet_rpc_server::validate_transfer(const std::list<wallet_rpc::transfer_destination> destinations, std::string payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, epee::json_rpc::error& er)
   {
-    crypto::hash integrated_payment_id = cryptonote::null_hash;
+    crypto::hash8 integrated_payment_id = cryptonote::null_hash8;
+    std::string extra_nonce;
     for (auto it = destinations.begin(); it != destinations.end(); it++)
     {
       cryptonote::tx_destination_entry de;
       bool has_payment_id;
-      crypto::hash new_payment_id;
+      crypto::hash8 new_payment_id;
       if(!get_account_integrated_address_from_str(de.addr, has_payment_id, new_payment_id, m_wallet.testnet(), it->address))
       {
         er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS;
@@ -136,13 +137,14 @@ namespace tools
 
       if (has_payment_id)
       {
-        if (!payment_id.empty() || integrated_payment_id != cryptonote::null_hash)
+        if (!payment_id.empty() || integrated_payment_id != cryptonote::null_hash8)
         {
           er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID;
           er.message = "A single payment id is allowed per transaction";
           return false;
         }
         integrated_payment_id = new_payment_id;
+        cryptonote::set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, integrated_payment_id);
       }
     }
 
@@ -152,17 +154,23 @@ namespace tools
       /* Just to clarify */
       const std::string& payment_id_str = payment_id;
 
-      crypto::hash payment_id;
+      crypto::hash long_payment_id;
+      crypto::hash8 short_payment_id;
+
       /* Parse payment ID */
-      if (!wallet2::parse_payment_id(payment_id_str, payment_id)) {
+      if (wallet2::parse_long_payment_id(payment_id_str, long_payment_id)) {
+        cryptonote::set_payment_id_to_tx_extra_nonce(extra_nonce, long_payment_id);
+      }
+      /* or short payment ID */
+      else if (!wallet2::parse_short_payment_id(payment_id_str, short_payment_id)) {
+        cryptonote::set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, short_payment_id);
+      }
+      else {
         er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID;
-        er.message = "Payment id has invalid format: \"" + payment_id_str + "\", expected 64-character string";
+        er.message = "Payment id has invalid format: \"" + payment_id_str + "\", expected 16 or 64 character string";
         return false;
       }
 
-      std::string extra_nonce;
-      cryptonote::set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id);
-      
       /* Append Payment ID data into extra */
       if (!cryptonote::add_extra_nonce_to_tx_extra(extra, extra_nonce)) {
         er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID;
@@ -339,14 +347,14 @@ namespace tools
   {
     try
     {
-      crypto::hash payment_id;
+      crypto::hash8 payment_id;
       if (req.payment_id.empty())
       {
-        crypto::generate_random_bytes(32, payment_id.data);
+        crypto::generate_random_bytes(8, payment_id.data);
       }
       else
       {
-        if (!tools::wallet2::parse_payment_id(req.payment_id,payment_id))
+        if (!tools::wallet2::parse_short_payment_id(req.payment_id,payment_id))
         {
           er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID;
           er.message = "Invalid payment ID";
@@ -371,7 +379,7 @@ namespace tools
     try
     {
       cryptonote::account_public_address address;
-      crypto::hash payment_id;
+      crypto::hash8 payment_id;
       bool has_payment_id;
 
       if(!get_account_integrated_address_from_str(address, has_payment_id, payment_id, m_wallet.testnet(), req.integrated_address))
@@ -485,6 +493,7 @@ namespace tools
     for (auto & payment_id_str : req.payment_ids)
     {
       crypto::hash payment_id;
+      crypto::hash8 payment_id8;
       cryptonote::blobdata payment_id_blob;
 
       // TODO - should the whole thing fail because of one bad id?
@@ -496,15 +505,23 @@ namespace tools
         return false;
       }
 
-      if(sizeof(payment_id) != payment_id_blob.size())
+      if(sizeof(payment_id) == payment_id_blob.size())
+      {
+        payment_id = *reinterpret_cast<const crypto::hash*>(payment_id_blob.data());
+      }
+      else if(sizeof(payment_id8) == payment_id_blob.size())
+      {
+        payment_id8 = *reinterpret_cast<const crypto::hash8*>(payment_id_blob.data());
+        memcpy(payment_id.data, payment_id8.data, 8);
+        memset(payment_id.data + 8, 0, 24);
+      }
+      else
       {
         er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID;
         er.message = "Payment ID has invalid size: " + payment_id_str;
         return false;
       }
 
-      payment_id = *reinterpret_cast<const crypto::hash*>(payment_id_blob.data());
-
       std::list<wallet2::payment_details> payment_list;
       m_wallet.get_payments(payment_id, payment_list, req.min_block_height);