wallet: background sync with just the view key

- When background syncing, the wallet wipes the spend key
from memory and processes all new transactions. The wallet saves
all receives, spends, and "plausible" spends of receives the
wallet does not know key images for.
- When background sync disabled, the wallet processes all
background synced txs and then clears the background sync cache.
- Adding "plausible" spends to the background sync cache ensures
that the wallet does not need to query the daemon to see if any
received outputs were spent while background sync was enabled.
This would harm privacy especially for users of 3rd party daemons.
- To enable the feature in the CLI wallet, the user can set
background-sync to reuse-wallet-password or
custom-background-password and the wallet automatically syncs in
the background when the wallet locks, then processes all
background synced txs when the wallet is unlocked.
- The custom-background-password option enables the user to
open a distinct background wallet that only has a view key saved
and can be opened/closed/synced separately from the main wallet.
When the main wallet opens, it processes the background wallet's
cache.
- To enable the feature in the RPC wallet, there is a new
`/setup_background_sync` endpoint.
- HW, multsig and view-only wallets cannot background sync.

3 files changed, 264 insertions, 4 deletions

diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp
index 58cb84947..51e4f78ee 100644
--- a/src/wallet/api/wallet.cpp
+++ b/src/wallet/api/wallet.cpp
@@ -58,6 +58,40 @@ using namespace cryptonote;
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "WalletAPI"
 
+#define LOCK_REFRESH() \
+    bool refresh_enabled = m_refreshEnabled; \
+    m_refreshEnabled = false; \
+    m_wallet->stop(); \
+    m_refreshCV.notify_one(); \
+    boost::mutex::scoped_lock lock(m_refreshMutex); \
+    boost::mutex::scoped_lock lock2(m_refreshMutex2); \
+    epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ \
+        /* m_refreshMutex's still locked here */ \
+        if (refresh_enabled) \
+            startRefresh(); \
+    })
+
+#define PRE_VALIDATE_BACKGROUND_SYNC() \
+  do \
+  { \
+    clearStatus(); \
+    if (m_wallet->key_on_device()) \
+    { \
+        setStatusError(tr("HW wallet cannot use background sync")); \
+        return false; \
+    } \
+    if (m_wallet->watch_only()) \
+    { \
+        setStatusError(tr("View only wallet cannot use background sync")); \
+        return false; \
+    } \
+    if (m_wallet->get_multisig_status().multisig_is_active) \
+    { \
+        setStatusError(tr("Multisig wallet cannot use background sync")); \
+        return false; \
+    } \
+  } while (0)
+
 namespace Monero {
 
 namespace {
@@ -766,6 +800,8 @@ bool WalletImpl::close(bool store)
 
 std::string WalletImpl::seed(const std::string& seed_offset) const
 {
+    if (checkBackgroundSync("cannot get seed"))
+        return std::string();
     epee::wipeable_string seed;
     if (m_wallet)
         m_wallet->get_seed(seed, seed_offset);
@@ -779,6 +815,8 @@ std::string WalletImpl::getSeedLanguage() const
 
 void WalletImpl::setSeedLanguage(const std::string &arg)
 {
+    if (checkBackgroundSync("cannot set seed language"))
+        return;
     m_wallet->set_seed_language(arg);
 }
 
@@ -802,6 +840,8 @@ void WalletImpl::statusWithErrorString(int& status, std::string& errorString) co
 
 bool WalletImpl::setPassword(const std::string &password)
 {
+    if (checkBackgroundSync("cannot change password"))
+        return false;
     clearStatus();
     try {
         m_wallet->change_password(m_wallet->get_wallet_file(), m_password, password);
@@ -931,6 +971,8 @@ bool WalletImpl::init(const std::string &daemon_address, uint64_t upper_transact
 
 void WalletImpl::setRefreshFromBlockHeight(uint64_t refresh_from_block_height)
 {
+    if (checkBackgroundSync("cannot change refresh height"))
+        return;
     m_wallet->set_refresh_from_block_height(refresh_from_block_height);
 }
 
@@ -1039,6 +1081,8 @@ void WalletImpl::refreshAsync()
 
 bool WalletImpl::rescanBlockchain()
 {
+    if (checkBackgroundSync("cannot rescan blockchain"))
+        return false;
     clearStatus();
     m_refreshShouldRescan = true;
     doRefresh();
@@ -1047,6 +1091,8 @@ bool WalletImpl::rescanBlockchain()
 
 void WalletImpl::rescanBlockchainAsync()
 {
+    if (checkBackgroundSync("cannot rescan blockchain"))
+        return;
     m_refreshShouldRescan = true;
     refreshAsync();
 }
@@ -1070,7 +1116,7 @@ int WalletImpl::autoRefreshInterval() const
 UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_filename) {
   clearStatus();
   UnsignedTransactionImpl * transaction = new UnsignedTransactionImpl(*this);
-  if (!m_wallet->load_unsigned_tx(unsigned_filename, transaction->m_unsigned_tx_set)){
+  if (checkBackgroundSync("cannot load tx") || !m_wallet->load_unsigned_tx(unsigned_filename, transaction->m_unsigned_tx_set)){
     setStatusError(tr("Failed to load unsigned transactions"));
     transaction->m_status = UnsignedTransaction::Status::Status_Error;
     transaction->m_errorString = errorString();
@@ -1090,6 +1136,8 @@ UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_file
 
 bool WalletImpl::submitTransaction(const string &fileName) {
   clearStatus();
+  if (checkBackgroundSync("cannot submit tx"))
+    return false;
   std::unique_ptr<PendingTransactionImpl> transaction(new PendingTransactionImpl(*this));
 
   bool r = m_wallet->load_tx(fileName, transaction->m_pending_tx);
@@ -1113,6 +1161,8 @@ bool WalletImpl::exportKeyImages(const string &filename, bool all)
     setStatusError(tr("Wallet is view only"));
     return false;
   }
+  if (checkBackgroundSync("cannot export key images"))
+    return false;
   
   try
   {
@@ -1133,6 +1183,8 @@ bool WalletImpl::exportKeyImages(const string &filename, bool all)
 
 bool WalletImpl::importKeyImages(const string &filename)
 {
+  if (checkBackgroundSync("cannot import key images"))
+    return false;
   if (!trustedDaemon()) {
     setStatusError(tr("Key images can only be imported with a trusted daemon"));
     return false;
@@ -1156,6 +1208,8 @@ bool WalletImpl::importKeyImages(const string &filename)
 
 bool WalletImpl::exportOutputs(const string &filename, bool all)
 {
+    if (checkBackgroundSync("cannot export outputs"))
+        return false;
     if (m_wallet->key_on_device())
     {
         setStatusError(string(tr("Not supported on HW wallets.")) + filename);
@@ -1186,6 +1240,8 @@ bool WalletImpl::exportOutputs(const string &filename, bool all)
 
 bool WalletImpl::importOutputs(const string &filename)
 {
+    if (checkBackgroundSync("cannot import outputs"))
+        return false;
     if (m_wallet->key_on_device())
     {
         setStatusError(string(tr("Not supported on HW wallets.")) + filename);
@@ -1218,6 +1274,8 @@ bool WalletImpl::importOutputs(const string &filename)
 
 bool WalletImpl::scanTransactions(const std::vector<std::string> &txids)
 {
+    if (checkBackgroundSync("cannot scan transactions"))
+        return false;
     if (txids.empty())
     {
         setStatusError(string(tr("Failed to scan transactions: no transaction ids provided.")));
@@ -1256,8 +1314,86 @@ bool WalletImpl::scanTransactions(const std::vector<std::string> &txids)
     return true;
 }
 
+bool WalletImpl::setupBackgroundSync(const Wallet::BackgroundSyncType background_sync_type, const std::string &wallet_password, const optional<std::string> &background_cache_password)
+{
+    try
+    {
+        PRE_VALIDATE_BACKGROUND_SYNC();
+
+        tools::wallet2::BackgroundSyncType bgs_type;
+        switch (background_sync_type)
+        {
+            case Wallet::BackgroundSync_Off: bgs_type = tools::wallet2::BackgroundSyncOff; break;
+            case Wallet::BackgroundSync_ReusePassword: bgs_type = tools::wallet2::BackgroundSyncReusePassword; break;
+            case Wallet::BackgroundSync_CustomPassword: bgs_type = tools::wallet2::BackgroundSyncCustomPassword; break;
+            default: setStatusError(tr("Unknown background sync type")); return false;
+        }
+
+        boost::optional<epee::wipeable_string> bgc_password = background_cache_password
+            ? boost::optional<epee::wipeable_string>(*background_cache_password)
+            : boost::none;
+
+        LOCK_REFRESH();
+        m_wallet->setup_background_sync(bgs_type, wallet_password, bgc_password);
+    }
+    catch (const std::exception &e)
+    {
+        LOG_ERROR("Failed to setup background sync: " << e.what());
+        setStatusError(string(tr("Failed to setup background sync: ")) + e.what());
+        return false;
+    }
+    return true;
+}
+
+Wallet::BackgroundSyncType WalletImpl::getBackgroundSyncType() const
+{
+    switch (m_wallet->background_sync_type())
+    {
+        case tools::wallet2::BackgroundSyncOff: return Wallet::BackgroundSync_Off;
+        case tools::wallet2::BackgroundSyncReusePassword: return Wallet::BackgroundSync_ReusePassword;
+        case tools::wallet2::BackgroundSyncCustomPassword: return Wallet::BackgroundSync_CustomPassword;
+        default: setStatusError(tr("Unknown background sync type")); return Wallet::BackgroundSync_Off;
+    }
+}
+
+bool WalletImpl::startBackgroundSync()
+{
+    try
+    {
+        PRE_VALIDATE_BACKGROUND_SYNC();
+        LOCK_REFRESH();
+        m_wallet->start_background_sync();
+    }
+    catch (const std::exception &e)
+    {
+        LOG_ERROR("Failed to start background sync: " << e.what());
+        setStatusError(string(tr("Failed to start background sync: ")) + e.what());
+        return false;
+    }
+    return true;
+}
+
+bool WalletImpl::stopBackgroundSync(const std::string &wallet_password)
+{
+    try
+    {
+        PRE_VALIDATE_BACKGROUND_SYNC();
+        LOCK_REFRESH();
+        m_wallet->stop_background_sync(epee::wipeable_string(wallet_password));
+    }
+    catch (const std::exception &e)
+    {
+        LOG_ERROR("Failed to stop background sync: " << e.what());
+        setStatusError(string(tr("Failed to stop background sync: ")) + e.what());
+        return false;
+    }
+    return true;
+}
+
 void WalletImpl::addSubaddressAccount(const std::string& label)
 {
+    if (checkBackgroundSync("cannot add account"))
+        return;
     m_wallet->add_subaddress_account(label);
 }
 size_t WalletImpl::numSubaddressAccounts() const
@@ -1270,10 +1406,14 @@ size_t WalletImpl::numSubaddresses(uint32_t accountIndex) const
 }
 void WalletImpl::addSubaddress(uint32_t accountIndex, const std::string& label)
 {
+    if (checkBackgroundSync("cannot add subbaddress"))
+        return;
     m_wallet->add_subaddress(accountIndex, label);
 }
 std::string WalletImpl::getSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex) const
 {
+    if (checkBackgroundSync("cannot get subbaddress label"))
+        return "";
     try
     {
         return m_wallet->get_subaddress_label({accountIndex, addressIndex});
@@ -1287,6 +1427,8 @@ std::string WalletImpl::getSubaddressLabel(uint32_t accountIndex, uint32_t addre
 }
 void WalletImpl::setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label)
 {
+    if (checkBackgroundSync("cannot set subbaddress label"))
+        return;
     try
     {
         return m_wallet->set_subaddress_label({accountIndex, addressIndex}, label);
@@ -1300,6 +1442,9 @@ void WalletImpl::setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex
 
 MultisigState WalletImpl::multisig() const {
     MultisigState state;
+    if (checkBackgroundSync("cannot use multisig"))
+        return state;
+
     const multisig::multisig_account_status ms_status{m_wallet->get_multisig_status()};
 
     state.isMultisig = ms_status.multisig_is_active;
@@ -1312,6 +1457,8 @@ MultisigState WalletImpl::multisig() const {
 }
 
 string WalletImpl::getMultisigInfo() const {
+    if (checkBackgroundSync("cannot use multisig"))
+        return string();
     try {
         clearStatus();
         return m_wallet->get_multisig_first_kex_msg();
@@ -1324,6 +1471,8 @@ string WalletImpl::getMultisigInfo() const {
 }
 
 string WalletImpl::makeMultisig(const vector<string>& info, const uint32_t threshold) {
+    if (checkBackgroundSync("cannot make multisig"))
+        return string();
     try {
         clearStatus();
 
@@ -1464,6 +1613,9 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri
     PendingTransactionImpl * transaction = new PendingTransactionImpl(*this);
 
     do {
+        if (checkBackgroundSync("cannot create transactions"))
+            break;
+
         std::vector<uint8_t> extra;
         std::string extra_nonce;
         vector<cryptonote::tx_destination_entry> dsts;
@@ -1630,6 +1782,9 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction()
     PendingTransactionImpl * transaction = new PendingTransactionImpl(*this);
 
     do {
+        if (checkBackgroundSync("cannot sweep"))
+            break;
+
         try {
             transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions();
             pendingTxPostProcess(transaction);
@@ -1763,11 +1918,15 @@ uint32_t WalletImpl::defaultMixin() const
 
 void WalletImpl::setDefaultMixin(uint32_t arg)
 {
+    if (checkBackgroundSync("cannot set default mixin"))
+        return;
     m_wallet->default_mixin(arg);
 }
 
 bool WalletImpl::setCacheAttribute(const std::string &key, const std::string &val)
 {
+    if (checkBackgroundSync("cannot set cache attribute"))
+        return false;
     m_wallet->set_attribute(key, val);
     return true;
 }
@@ -1781,6 +1940,8 @@ std::string WalletImpl::getCacheAttribute(const std::string &key) const
 
 bool WalletImpl::setUserNote(const std::string &txid, const std::string &note)
 {
+    if (checkBackgroundSync("cannot set user note"))
+        return false;
     cryptonote::blobdata txid_data;
     if(!epee::string_tools::parse_hexstr_to_binbuff(txid, txid_data) || txid_data.size() != sizeof(crypto::hash))
       return false;
@@ -1792,6 +1953,8 @@ bool WalletImpl::setUserNote(const std::string &txid, const std::string &note)
 
 std::string WalletImpl::getUserNote(const std::string &txid) const
 {
+    if (checkBackgroundSync("cannot get user note"))
+        return "";
     cryptonote::blobdata txid_data;
     if(!epee::string_tools::parse_hexstr_to_binbuff(txid, txid_data) || txid_data.size() != sizeof(crypto::hash))
       return "";
@@ -1802,6 +1965,9 @@ std::string WalletImpl::getUserNote(const std::string &txid) const
 
 std::string WalletImpl::getTxKey(const std::string &txid_str) const
 {
+    if (checkBackgroundSync("cannot get tx key"))
+        return "";
+
     crypto::hash txid;
     if(!epee::string_tools::hex_to_pod(txid_str, txid))
     {
@@ -1886,6 +2052,9 @@ bool WalletImpl::checkTxKey(const std::string &txid_str, std::string tx_key_str,
 
 std::string WalletImpl::getTxProof(const std::string &txid_str, const std::string &address_str, const std::string &message) const
 {
+    if (checkBackgroundSync("cannot get tx proof"))
+        return "";
+
     crypto::hash txid;
     if (!epee::string_tools::hex_to_pod(txid_str, txid))
     {
@@ -1942,6 +2111,9 @@ bool WalletImpl::checkTxProof(const std::string &txid_str, const std::string &ad
 }
 
 std::string WalletImpl::getSpendProof(const std::string &txid_str, const std::string &message) const {
+    if (checkBackgroundSync("cannot get spend proof"))
+        return "";
+
     crypto::hash txid;
     if(!epee::string_tools::hex_to_pod(txid_str, txid))
     {
@@ -1984,6 +2156,9 @@ bool WalletImpl::checkSpendProof(const std::string &txid_str, const std::string
 }
 
 std::string WalletImpl::getReserveProof(bool all, uint32_t account_index, uint64_t amount, const std::string &message) const {
+    if (checkBackgroundSync("cannot get reserve proof"))
+        return "";
+
     try
     {
         clearStatus();
@@ -2030,6 +2205,9 @@ bool WalletImpl::checkReserveProof(const std::string &address, const std::string
 
 std::string WalletImpl::signMessage(const std::string &message, const std::string &address)
 {
+    if (checkBackgroundSync("cannot sign message"))
+        return "";
+
     if (address.empty()) {
         return m_wallet->sign(message, tools::wallet2::sign_with_spend_key);
     }
@@ -2156,6 +2334,16 @@ bool WalletImpl::isDeterministic() const
     return m_wallet->is_deterministic();
 }
 
+bool WalletImpl::isBackgroundSyncing() const
+{
+    return m_wallet->is_background_syncing();
+}
+
+bool WalletImpl::isBackgroundWallet() const
+{
+    return m_wallet->is_background_wallet();
+}
+
 void WalletImpl::clearStatus() const
 {
     boost::lock_guard<boost::mutex> l(m_statusMutex);
@@ -2224,9 +2412,7 @@ void WalletImpl::doRefresh()
             if(rescan)
                 m_wallet->rescan_blockchain(false);
             m_wallet->refresh(trustedDaemon());
-            if (!m_synchronized) {
-                m_synchronized = true;
-            }
+            m_synchronized = m_wallet->is_synced();
             // assuming if we have empty history, it wasn't initialized yet
             // for further history changes client need to update history in
             // "on_money_received" and "on_money_sent" callbacks
@@ -2329,6 +2515,24 @@ bool WalletImpl::doInit(const string &daemon_address, const std::string &proxy_a
     return true;
 }
 
+bool WalletImpl::checkBackgroundSync(const std::string &message) const
+{
+    clearStatus();
+    if (m_wallet->is_background_wallet())
+    {
+        LOG_ERROR("Background wallets " + message);
+        setStatusError(tr("Background wallets ") + message);
+        return true;
+    }
+    if (m_wallet->is_background_syncing())
+    {
+        LOG_ERROR(message + " while background syncing");
+        setStatusError(message + tr(" while background syncing. Stop background syncing first."));
+        return true;
+    }
+    return false;
+}
+
 bool WalletImpl::parse_uri(const std::string &uri, std::string &address, std::string &payment_id, uint64_t &amount, std::string &tx_description, std::string &recipient_name, std::vector<std::string> &unknown_parameters, std::string &error)
 {
     return m_wallet->parse_uri(uri, address, payment_id, amount, tx_description, recipient_name, unknown_parameters, error);
@@ -2347,6 +2551,8 @@ std::string WalletImpl::getDefaultDataDir() const
 bool WalletImpl::rescanSpent()
 {
   clearStatus();
+  if (checkBackgroundSync("cannot rescan spent"))
+    return false;
   if (!trustedDaemon()) {
     setStatusError(tr("Rescan spent can only be used with a trusted daemon"));
     return false;
diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h
index d1bf4f759..32a19ec07 100644
--- a/src/wallet/api/wallet.h
+++ b/src/wallet/api/wallet.h
@@ -172,6 +172,13 @@ public:
     bool importOutputs(const std::string &filename) override;
     bool scanTransactions(const std::vector<std::string> &txids) override;
 
+    bool setupBackgroundSync(const BackgroundSyncType background_sync_type, const std::string &wallet_password, const optional<std::string> &background_cache_password = optional<std::string>()) override;
+    BackgroundSyncType getBackgroundSyncType() const override;
+    bool startBackgroundSync() override;
+    bool stopBackgroundSync(const std::string &wallet_password) override;
+    bool isBackgroundSyncing() const override;
+    bool isBackgroundWallet() const override;
+
     virtual void disposeTransaction(PendingTransaction * t) override;
     virtual uint64_t estimateTransactionFee(const std::vector<std::pair<std::string, uint64_t>> &destinations,
                                             PendingTransaction::Priority priority) const override;
@@ -238,6 +245,7 @@ private:
     bool isNewWallet() const;
     void pendingTxPostProcess(PendingTransactionImpl * pending);
     bool doInit(const std::string &daemon_address, const std::string &proxy_address, uint64_t upper_transaction_size_limit = 0, bool ssl = false);
+    bool checkBackgroundSync(const std::string &message) const;
 
 private:
     friend class PendingTransactionImpl;
@@ -253,6 +261,10 @@ private:
     mutable boost::mutex m_statusMutex;
     mutable int m_status;
     mutable std::string m_errorString;
+    // TODO: harden password handling in the wallet API, see relevant discussion
+    // https://github.com/monero-project/monero-gui/issues/1537
+    // https://github.com/feather-wallet/feather/issues/72#issuecomment-1405602142
+    // https://github.com/monero-project/monero/pull/8619#issuecomment-1632951461
     std::string m_password;
     std::unique_ptr<TransactionHistoryImpl> m_history;
     std::unique_ptr<Wallet2CallbackImpl> m_wallet2Callback;
diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h
index 53210832b..be84003fa 100644
--- a/src/wallet/api/wallet2_api.h
+++ b/src/wallet/api/wallet2_api.h
@@ -446,6 +446,12 @@ struct Wallet
         ConnectionStatus_WrongVersion
     };
 
+    enum BackgroundSyncType {
+        BackgroundSync_Off = 0,
+        BackgroundSync_ReusePassword = 1,
+        BackgroundSync_CustomPassword = 2
+    };
+
     virtual ~Wallet() = 0;
     virtual std::string seed(const std::string& seed_offset = "") const = 0;
     virtual std::string getSeedLanguage() const = 0;
@@ -937,6 +943,42 @@ struct Wallet
      */
     virtual bool scanTransactions(const std::vector<std::string> &txids) = 0;
 
+    /*!
+     * \brief setupBackgroundSync       - setup background sync mode with just a view key
+     * \param background_sync_type      - the mode the wallet background syncs in
+     * \param wallet_password
+     * \param background_cache_password - custom password to encrypt background cache, only needed for custom password background sync type
+     * \return                          - true on success
+     */
+    virtual bool setupBackgroundSync(const BackgroundSyncType background_sync_type, const std::string &wallet_password, const optional<std::string> &background_cache_password) = 0;
+
+    /*!
+     * \brief getBackgroundSyncType     - get mode the wallet background syncs in
+     * \return                          - the type, or off if type is unknown
+     */
+    virtual BackgroundSyncType getBackgroundSyncType() const = 0;
+
+    /**
+     * @brief startBackgroundSync - sync the chain in the background with just view key
+     */
+    virtual bool startBackgroundSync() = 0;
+
+    /**
+     * @brief stopBackgroundSync  - bring back spend key and process background synced txs
+     * \param wallet_password
+     */
+    virtual bool stopBackgroundSync(const std::string &wallet_password) = 0;
+
+    /**
+     * @brief isBackgroundSyncing - returns true if the wallet is background syncing
+     */
+    virtual bool isBackgroundSyncing() const = 0;
+
+    /**
+     * @brief isBackgroundWallet - returns true if the wallet is a background wallet
+     */
+    virtual bool isBackgroundWallet() const = 0;
+
     virtual TransactionHistory * history() = 0;
     virtual AddressBook * addressBook() = 0;
     virtual Subaddress * subaddress() = 0;