Merge pull request #497

9b945f5 wallet: make the refresh optimizations selectable via command line (moneromooo-monero)
d2c0313 wallet: speedup refresh from daemon (moneromooo-monero)
ea707c7 wallet2: minor cleanup (moneromooo-monero)
55a2da7 wallet2: speedup refresh a bit (moneromooo-monero)

4 files changed, 262 insertions, 40 deletions

diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp
index d3669965f..2d99b3da0 100644
--- a/src/simplewallet/simplewallet.cpp
+++ b/src/simplewallet/simplewallet.cpp
@@ -93,6 +93,7 @@ namespace
   const command_line::arg_descriptor<bool> arg_testnet = {"testnet", sw::tr("Used to deploy test nets. The daemon must be launched with --testnet flag"), false};
   const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", sw::tr("Restricts RPC to view only commands"), false};
   const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", sw::tr("Enable commands which rely on a trusted daemon"), false};
+  const command_line::arg_descriptor<std::string> arg_refresh_type = {"refresh-type", sw::tr("Control the wallet refresh speedup/assumptions balance: full (slowest, no assumptions), optimize-coinbase (fast, assumes the whole coinbase is paid to a single address), no-coinbase (fastest, assumes we receive no coinbase transaction)"), "optimize-coinbase"};
 
   const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""};
 
@@ -631,7 +632,8 @@ void simple_wallet::print_seed(std::string seed)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::init(const boost::program_options::variables_map& vm)
 {
-  handle_command_line(vm);
+  if (!handle_command_line(vm))
+    return false;
 
   if (!m_daemon_address.empty() && !m_daemon_host.empty() && 0 != m_daemon_port)
   {
@@ -769,7 +771,32 @@ bool simple_wallet::deinit()
   return close_wallet();
 }
 //----------------------------------------------------------------------------------------------------
-void simple_wallet::handle_command_line(const boost::program_options::variables_map& vm)
+static bool parse_refresh_type(const std::string &s, tools::wallet2::RefreshType &refresh_type)
+{
+  static const struct
+  {
+    const char *name;
+    tools::wallet2::RefreshType refresh_type;
+  } names[] =
+  {
+    { "full", tools::wallet2::RefreshFull },
+    { "optimize-coinbase", tools::wallet2::RefreshOptimizeCoinbase },
+    { "optimized-coinbase", tools::wallet2::RefreshOptimizeCoinbase },
+    { "no-coinbase", tools::wallet2::RefreshNoCoinbase },
+  };
+  for (size_t n = 0; n < sizeof(names) / sizeof(names[0]); ++n)
+  {
+    if (s == names[n].name)
+    {
+      refresh_type = names[n].refresh_type;
+      return true;
+    }
+  }
+  fail_msg_writer() << tr("Failed to parse refresh type");
+  return false;
+}
+//----------------------------------------------------------------------------------------------------
+bool simple_wallet::handle_command_line(const boost::program_options::variables_map& vm)
 {
   m_wallet_file                   = command_line::get_arg(vm, arg_wallet_file);
   m_generate_new                  = command_line::get_arg(vm, arg_generate_new_wallet);
@@ -781,6 +808,12 @@ void simple_wallet::handle_command_line(const boost::program_options::variables_
   m_restore_deterministic_wallet  = command_line::get_arg(vm, arg_restore_deterministic_wallet);
   m_non_deterministic             = command_line::get_arg(vm, arg_non_deterministic);
   m_trusted_daemon                = command_line::get_arg(vm, arg_trusted_daemon);
+  std::string refresh_type        = command_line::get_arg(vm, arg_refresh_type);
+
+  if (!parse_refresh_type(refresh_type, m_refresh_type))
+    return false;
+
+  return true;
 }
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::try_connect_to_daemon()
@@ -863,6 +896,7 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
   m_wallet.reset(new tools::wallet2(testnet));
   m_wallet->callback(this);
   m_wallet->set_seed_language(mnemonic_language);
+  m_wallet->set_refresh_type(m_refresh_type);
 
   crypto::secret_key recovery_val;
   try
@@ -911,6 +945,7 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
 
   m_wallet.reset(new tools::wallet2(testnet));
   m_wallet->callback(this);
+  m_wallet->set_refresh_type(m_refresh_type);
 
   try
   {
@@ -941,6 +976,7 @@ bool simple_wallet::open_wallet(const string &wallet_file, const std::string& pa
   m_wallet_file = wallet_file;
   m_wallet.reset(new tools::wallet2(testnet));
   m_wallet->callback(this);
+  m_wallet->set_refresh_type(m_refresh_type);
 
   try
   {
@@ -2229,6 +2265,7 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_params, arg_testnet);
   command_line::add_arg(desc_params, arg_restricted);
   command_line::add_arg(desc_params, arg_trusted_daemon);
+  command_line::add_arg(desc_params, arg_refresh_type);
   tools::wallet_rpc_server::init_options(desc_params);
 
   po::positional_options_description positional_options;
diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h
index 94ad724be..2ad54d4ca 100644
--- a/src/simplewallet/simplewallet.h
+++ b/src/simplewallet/simplewallet.h
@@ -73,7 +73,7 @@ namespace cryptonote
     bool process_command(const std::vector<std::string> &args);
     std::string get_commands_str();
   private:
-    void handle_command_line(const boost::program_options::variables_map& vm);
+    bool handle_command_line(const boost::program_options::variables_map& vm);
 
     bool run_console_handler();
 
@@ -222,6 +222,8 @@ namespace cryptonote
     std::string m_daemon_host;
     int m_daemon_port;
 
+    tools::wallet2::RefreshType m_refresh_type;
+
     epee::console_handlers_binder m_cmd_binder;
 
     std::unique_ptr<tools::wallet2> m_wallet;
diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index f26809af8..a6ac860c7 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -68,6 +68,13 @@ using namespace cryptonote;
 // arbitrary, used to generate different hashes from the same input
 #define CHACHA8_KEY_TAIL 0x8c
 
+#define KILL_IOSERVICE()  \
+    do { \
+      work.reset(); \
+      threadpool.join_all(); \
+      ioservice.stop(); \
+    } while(0)
+
 namespace
 {
 void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file)
@@ -148,9 +155,29 @@ bool wallet2::is_deprecated() const
   return is_old_file_format;
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_t height)
+void wallet2::check_acc_out(const account_keys &acc, const tx_out &o, const crypto::public_key &tx_pub_key, size_t i, uint64_t &money_transfered, bool &error) const
+{
+  if (o.target.type() !=  typeid(txout_to_key))
+  {
+     error = true;
+     LOG_ERROR("wrong type id in transaction out");
+     return;
+  }
+  if(is_out_to_acc(acc, boost::get<txout_to_key>(o.target), tx_pub_key, i))
+  {
+    money_transfered = o.amount;
+  }
+  else
+  {
+    money_transfered = 0;
+  }
+  error = false;
+}
+//----------------------------------------------------------------------------------------------------
+void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_t height, bool miner_tx)
 {
-  process_unconfirmed(tx, height);
+  if (!miner_tx)
+    process_unconfirmed(tx, height);
   std::vector<size_t> outs;
   uint64_t tx_money_got_in_outs = 0;
   crypto::public_key tx_pub_key = null_pkey;
@@ -175,7 +202,69 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_
     }
 
     tx_pub_key = pub_key_field.pub_key;
-    bool r = lookup_acc_outs(m_account.get_keys(), tx, tx_pub_key, outs, tx_money_got_in_outs);
+    bool r = true;
+    int threads;
+    if (miner_tx && m_refresh_type == RefreshNoCoinbase)
+    {
+      // assume coinbase isn't for us
+    }
+    else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase)
+    {
+      for (size_t i = 0; i < tx.vout.size(); ++i)
+      {
+        uint64_t money_transfered = 0;
+        bool error = false;
+        check_acc_out(m_account.get_keys(), tx.vout[i], tx_pub_key, i, money_transfered, error);
+        if (error)
+        {
+          r = false;
+          break;
+        }
+        // this assumes that the miner tx pays a single address
+        if (money_transfered == 0)
+          break;
+        outs.push_back(i);
+        tx_money_got_in_outs += money_transfered;
+      }
+    }
+    else if (tx.vout.size() > 1 && (threads = std::thread::hardware_concurrency()) > 1)
+    {
+      boost::asio::io_service ioservice;
+      boost::thread_group threadpool;
+      std::auto_ptr < boost::asio::io_service::work > work(new boost::asio::io_service::work(ioservice));
+      for (int i = 0; i < threads; i++)
+      {
+        threadpool.create_thread(boost::bind(&boost::asio::io_service::run, &ioservice));
+      }
+
+      const account_keys &keys = m_account.get_keys();
+      std::vector<uint64_t> money_transfered(tx.vout.size());
+      std::deque<bool> error(tx.vout.size());
+      for (size_t i = 0; i < tx.vout.size(); ++i)
+      {
+        ioservice.dispatch(boost::bind(&wallet2::check_acc_out, this, std::cref(keys), std::cref(tx.vout[i]), std::cref(tx_pub_key), i,
+          std::ref(money_transfered[i]), std::ref(error[i])));
+      }
+      KILL_IOSERVICE();
+      tx_money_got_in_outs = 0;
+      for (size_t i = 0; i < tx.vout.size(); ++i)
+      {
+        if (error[i])
+        {
+          r = false;
+          break;
+        }
+        if (money_transfered[i])
+        {
+          outs.push_back(i);
+          tx_money_got_in_outs += money_transfered[i];
+        }
+      }
+    }
+    else
+    {
+      r = lookup_acc_outs(m_account.get_keys(), tx, tx_pub_key, outs, tx_money_got_in_outs);
+    }
     THROW_WALLET_EXCEPTION_IF(!r, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys());
 
     if(!outs.empty() && tx_money_got_in_outs)
@@ -236,50 +325,53 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_
     }
   }
 
-  tx_extra_nonce extra_nonce;
-  crypto::hash payment_id = null_hash;
-  if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce))
+  if (tx_money_spent_in_ins > 0)
   {
-    crypto::hash8 payment_id8 = null_hash8;
-    if(get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8))
+    process_outgoing(tx, height, tx_money_spent_in_ins, tx_money_got_in_outs);
+  }
+
+  uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0;
+  if (0 < received)
+  {
+    tx_extra_nonce extra_nonce;
+    crypto::hash payment_id = null_hash;
+    if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce))
     {
-      // We got a payment ID to go with this tx
-      LOG_PRINT_L2("Found encrypted payment ID: " << payment_id8);
-      if (tx_pub_key != null_pkey)
+      crypto::hash8 payment_id8 = null_hash8;
+      if(get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8))
       {
-        if (!decrypt_payment_id(payment_id8, tx_pub_key, m_account.get_keys().m_view_secret_key))
+        // We got a payment ID to go with this tx
+        LOG_PRINT_L2("Found encrypted payment ID: " << payment_id8);
+        if (tx_pub_key != null_pkey)
         {
-          LOG_PRINT_L0("Failed to decrypt payment ID: " << payment_id8);
+          if (!decrypt_payment_id(payment_id8, tx_pub_key, m_account.get_keys().m_view_secret_key))
+          {
+            LOG_PRINT_L0("Failed to decrypt payment ID: " << payment_id8);
+          }
+          else
+          {
+            LOG_PRINT_L2("Decrypted payment ID: " << payment_id8);
+            // put the 64 bit decrypted payment id in the first 8 bytes
+            memcpy(payment_id.data, payment_id8.data, 8);
+            // rest is already 0, but guard against code changes above
+            memset(payment_id.data + 8, 0, 24);
+          }
         }
         else
         {
-          LOG_PRINT_L2("Decrypted payment ID: " << payment_id8);
-          // put the 64 bit decrypted payment id in the first 8 bytes
-          memcpy(payment_id.data, payment_id8.data, 8);
-          // rest is already 0, but guard against code changes above
-          memset(payment_id.data + 8, 0, 24);
+          LOG_PRINT_L1("No public key found in tx, unable to decrypt payment id");
         }
       }
-      else
+      else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id))
       {
-        LOG_PRINT_L1("No public key found in tx, unable to decrypt payment id");
+        LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id);
       }
     }
     else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id))
     {
       LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id);
     }
-  }
-
-  uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0;
 
-  if (tx_money_spent_in_ins > 0)
-  {
-    process_outgoing(tx, height, tx_money_spent_in_ins, tx_money_got_in_outs);
-  }
-
-  if (0 < received)
-  {
     payment_details payment;
     payment.m_tx_hash      = cryptonote::get_transaction_hash(tx);
     payment.m_amount       = received;
@@ -320,7 +412,7 @@ void wallet2::process_outgoing(const cryptonote::transaction &tx, uint64_t heigh
   ctd.m_block_height = height;
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::process_new_blockchain_entry(const cryptonote::block& b, cryptonote::block_complete_entry& bche, crypto::hash& bl_id, uint64_t height)
+void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height)
 {
   //handle transactions from new block
     
@@ -328,7 +420,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, cryptonot
   if(b.timestamp + 60*60*24 > m_account.get_createtime())
   {
     TIME_MEASURE_START(miner_tx_handle_time);
-    process_new_transaction(b.miner_tx, height);
+    process_new_transaction(b.miner_tx, height, true);
     TIME_MEASURE_FINISH(miner_tx_handle_time);
 
     TIME_MEASURE_START(txs_handle_time);
@@ -337,7 +429,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, cryptonot
       cryptonote::transaction tx;
       bool r = parse_and_validate_tx_from_blob(txblob, tx);
       THROW_WALLET_EXCEPTION_IF(!r, error::tx_parse_error, txblob);
-      process_new_transaction(tx, height);
+      process_new_transaction(tx, height, false);
     }
     TIME_MEASURE_FINISH(txs_handle_time);
     LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " <<  miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms");
@@ -379,6 +471,13 @@ void wallet2::get_short_chain_history(std::list<crypto::hash>& ids) const
     ids.push_back(m_blockchain[0]);
 }
 //----------------------------------------------------------------------------------------------------
+void wallet2::parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const
+{
+  error = !cryptonote::parse_and_validate_block_from_blob(blob, bl);
+  if (!error)
+    bl_id = get_block_hash(bl);
+}
+//----------------------------------------------------------------------------------------------------
 void wallet2::pull_blocks(uint64_t start_height, uint64_t& blocks_added)
 {
   blocks_added = 0;
@@ -392,6 +491,74 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t& blocks_added)
   THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, res.status);
 
   size_t current_index = res.start_height;
+
+  int threads = std::thread::hardware_concurrency();
+  if (threads > 1)
+  {
+    std::vector<crypto::hash> round_block_hashes(threads);
+    std::vector<cryptonote::block> round_blocks(threads);
+    std::deque<bool> error(threads);
+    const std::list<block_complete_entry> &blocks = res.blocks;
+    size_t blocks_size = blocks.size();
+    std::list<block_complete_entry>::const_iterator blocki = blocks.begin();
+    for (size_t b = 0; b < blocks_size; b += threads)
+    {
+      size_t round_size = std::min((size_t)threads, blocks_size - b);
+
+      boost::asio::io_service ioservice;
+      boost::thread_group threadpool;
+      std::unique_ptr < boost::asio::io_service::work > work(new boost::asio::io_service::work(ioservice));
+      for (size_t i = 0; i < round_size; i++)
+      {
+        threadpool.create_thread(boost::bind(&boost::asio::io_service::run, &ioservice));
+      }
+
+      std::list<block_complete_entry>::const_iterator tmpblocki = blocki;
+      for (size_t i = 0; i < round_size; ++i)
+      {
+        ioservice.dispatch(boost::bind(&wallet2::parse_block_round, this, std::cref(tmpblocki->block),
+          std::ref(round_blocks[i]), std::ref(round_block_hashes[i]), std::ref(error[i])));
+        ++tmpblocki;
+      }
+      KILL_IOSERVICE();
+      tmpblocki = blocki;
+      for (size_t i = 0; i < round_size; ++i)
+      {
+        THROW_WALLET_EXCEPTION_IF(error[i], error::block_parse_error, tmpblocki->block);
+        ++tmpblocki;
+      }
+      for (size_t i = 0; i < round_size; ++i)
+      {
+        const crypto::hash &bl_id = round_block_hashes[i];
+        cryptonote::block &bl = round_blocks[i];
+
+        if(current_index >= m_blockchain.size())
+        {
+          process_new_blockchain_entry(bl, *blocki, bl_id, current_index);
+          ++blocks_added;
+        }
+        else if(bl_id != m_blockchain[current_index])
+        {
+          //split detected here !!!
+          THROW_WALLET_EXCEPTION_IF(current_index == start_height, error::wallet_internal_error,
+            "wrong daemon response: split starts from the first block in response " + string_tools::pod_to_hex(bl_id) +
+            " (height " + std::to_string(start_height) + "), local block id at this height: " +
+            string_tools::pod_to_hex(m_blockchain[current_index]));
+
+          detach_blockchain(current_index);
+          process_new_blockchain_entry(bl, *blocki, bl_id, current_index);
+        }
+        else
+        {
+          LOG_PRINT_L2("Block is already in blockchain: " << string_tools::pod_to_hex(bl_id));
+        }
+        ++current_index;
+        ++blocki;
+      }
+    }
+  }
+  else
+  {
   BOOST_FOREACH(auto& bl_entry, res.blocks)
   {
     cryptonote::block bl;
@@ -422,6 +589,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t& blocks_added)
 
     ++current_index;
   }
+  }
 }
 //----------------------------------------------------------------------------------------------------
 void wallet2::refresh()
diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h
index 8df6c757d..e036020b8 100644
--- a/src/wallet/wallet2.h
+++ b/src/wallet/wallet2.h
@@ -79,9 +79,18 @@ namespace tools
 
   class wallet2
   {
-    wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers (false), m_store_tx_info(true), m_default_mixin(0) {}
   public:
-    wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_restricted(restricted), is_old_file_format(false), m_store_tx_info(true), m_default_mixin(0) {}
+    enum RefreshType {
+      RefreshFull,
+      RefreshOptimizeCoinbase,
+      RefreshNoCoinbase,
+    };
+
+  private:
+    wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers (false), m_store_tx_info(true), m_default_mixin(0), m_refresh_type(RefreshOptimizeCoinbase) {}
+
+  public:
+    wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_restricted(restricted), is_old_file_format(false), m_store_tx_info(true), m_default_mixin(0), m_refresh_type(RefreshOptimizeCoinbase) {}
     struct transfer_details
     {
       uint64_t m_block_height;
@@ -234,6 +243,9 @@ namespace tools
     void refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money);
     bool refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok);
 
+    void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; }
+    RefreshType get_refresh_type(RefreshType refresh_type) const { return m_refresh_type; }
+
     bool testnet() const { return m_testnet; }
     bool restricted() const { return m_restricted; }
     bool watch_only() const { return m_watch_only; }
@@ -336,8 +348,8 @@ namespace tools
      * \param password       Password of wallet file
      */
     void load_keys(const std::string& keys_file_name, const std::string& password);
-    void process_new_transaction(const cryptonote::transaction& tx, uint64_t height);
-    void process_new_blockchain_entry(const cryptonote::block& b, cryptonote::block_complete_entry& bche, crypto::hash& bl_id, uint64_t height);
+    void process_new_transaction(const cryptonote::transaction& tx, uint64_t height, bool miner_tx);
+    void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height);
     void detach_blockchain(uint64_t height);
     void get_short_chain_history(std::list<crypto::hash>& ids) const;
     bool is_tx_spendtime_unlocked(uint64_t unlock_time) const;
@@ -353,6 +365,8 @@ namespace tools
     void check_genesis(const crypto::hash& genesis_hash) const; //throws
     bool generate_chacha8_key_from_secret_keys(crypto::chacha8_key &key) const;
     crypto::hash get_payment_id(const pending_tx &ptx) const;
+    void check_acc_out(const cryptonote::account_keys &acc, const cryptonote::tx_out &o, const crypto::public_key &tx_pub_key, size_t i, uint64_t &money_transfered, bool &error) const;
+    void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const;
 
     cryptonote::account_base m_account;
     std::string m_daemon_address;
@@ -382,6 +396,7 @@ namespace tools
     bool m_always_confirm_transfers;
     bool m_store_tx_info; /*!< request txkey to be returned in RPC, and store in the wallet cache file */
     uint32_t m_default_mixin;
+    RefreshType m_refresh_type;
   };
 }
 BOOST_CLASS_VERSION(tools::wallet2, 10)