allow exporting outputs in chunks

this will make it easier huge wallets to do so without hitting
random limits (eg, max string size in node).

5 files changed, 122 insertions, 46 deletions

diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp
index 1ee2e20b6..807dba33b 100644
--- a/src/wallet/api/wallet.cpp
+++ b/src/wallet/api/wallet.cpp
@@ -1146,8 +1146,8 @@ UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_file
   
   // Check tx data and construct confirmation message
   std::string extra_message;
-  if (!transaction->m_unsigned_tx_set.transfers.second.empty())
-    extra_message = (boost::format("%u outputs to import. ") % (unsigned)transaction->m_unsigned_tx_set.transfers.second.size()).str();
+  if (!std::get<2>(transaction->m_unsigned_tx_set.transfers).empty())
+    extra_message = (boost::format("%u outputs to import. ") % (unsigned)std::get<2>(transaction->m_unsigned_tx_set.transfers).size()).str();
   transaction->checkLoadedTx([&transaction](){return transaction->m_unsigned_tx_set.txes.size();}, [&transaction](size_t n)->const tools::wallet2::tx_construction_data&{return transaction->m_unsigned_tx_set.txes[n];}, extra_message);
   setStatus(transaction->status(), transaction->errorString());
     
diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index 536460396..05d86a817 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -6604,9 +6604,9 @@ bool wallet2::sign_tx(const std::string &unsigned_filename, const std::string &s
 //----------------------------------------------------------------------------------------------------
 bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &txs, signed_tx_set &signed_txes)
 {
-  if (!exported_txs.new_transfers.second.empty())
+  if (!std::get<2>(exported_txs.new_transfers).empty())
     import_outputs(exported_txs.new_transfers);
-  else if (!exported_txs.transfers.second.empty())
+  else if (!std::get<2>(exported_txs.transfers).empty())
     import_outputs(exported_txs.transfers);
 
   // sign the transactions
@@ -10800,7 +10800,7 @@ void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_
   {
     txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device()));
   }
-  txs.transfers = std::make_pair(0, m_transfers);
+  txs.transfers = std::make_tuple(0, m_transfers.size(), m_transfers);
 
   auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev);
   CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface");
@@ -13103,18 +13103,29 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect
   m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx);
 }
 //----------------------------------------------------------------------------------------------------
-std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> wallet2::export_outputs(bool all) const
+std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> wallet2::export_outputs(bool all, uint32_t start, uint32_t count) const
 {
   PERF_TIMER(export_outputs);
   std::vector<tools::wallet2::exported_transfer_details> outs;
 
+  // invalid cases
+  THROW_WALLET_EXCEPTION_IF(count == 0, error::wallet_internal_error, "Nothing requested");
+  THROW_WALLET_EXCEPTION_IF(!all && start > 0, error::wallet_internal_error, "Incremental mode is incompatible with non-zero start");
+
+  // valid cases:
+  // all: all outputs, subject to start/count
+  // !all: incremental, subject to count
+  // for convenience, start/count are allowed to go past the valid range, then nothing is returned
+
   size_t offset = 0;
   if (!all)
     while (offset < m_transfers.size() && (m_transfers[offset].m_key_image_known && !m_transfers[offset].m_key_image_request))
       ++offset;
+  else
+    offset = start;
 
   outs.reserve(m_transfers.size() - offset);
-  for (size_t n = offset; n < m_transfers.size(); ++n)
+  for (size_t n = offset; n < m_transfers.size() && n - offset < count; ++n)
   {
     const transfer_details &td = m_transfers[n];
 
@@ -13138,16 +13149,16 @@ std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> wall
     outs.push_back(etd);
   }
 
-  return std::make_pair(offset, outs);
+  return std::make_tuple(offset, m_transfers.size(), outs);
 }
 //----------------------------------------------------------------------------------------------------
-std::string wallet2::export_outputs_to_str(bool all) const
+std::string wallet2::export_outputs_to_str(bool all, uint32_t start, uint32_t count) const
 {
   PERF_TIMER(export_outputs_to_str);
 
   std::stringstream oss;
   binary_archive<true> ar(oss);
-  auto outputs = export_outputs(all);
+  auto outputs = export_outputs(all, start, count);
   THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, outputs), error::wallet_internal_error, "Failed to serialize output data");
 
   std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC));
@@ -13160,23 +13171,34 @@ std::string wallet2::export_outputs_to_str(bool all) const
   return magic + ciphertext;
 }
 //----------------------------------------------------------------------------------------------------
-size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs)
+size_t wallet2::import_outputs(const std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs)
 {
   PERF_TIMER(import_outputs);
 
   THROW_WALLET_EXCEPTION_IF(!m_offline, error::wallet_internal_error, "Hot wallets cannot import outputs");
 
-  THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error,
+  // we can now import piecemeal
+  const size_t offset = std::get<0>(outputs);
+  const size_t num_outputs = std::get<1>(outputs);
+  const std::vector<tools::wallet2::transfer_details> &output_array = std::get<2>(outputs);
+
+  THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error,
       "Imported outputs omit more outputs that we know of");
 
-  const size_t offset = outputs.first;
+  THROW_WALLET_EXCEPTION_IF(offset >= num_outputs, error::wallet_internal_error,
+      "Offset is larger than total outputs");
+  THROW_WALLET_EXCEPTION_IF(output_array.size() > num_outputs - offset, error::wallet_internal_error,
+      "Offset is larger than total outputs");
+
   const size_t original_size = m_transfers.size();
-  m_transfers.resize(offset + outputs.second.size());
-  for (size_t i = 0; i < offset; ++i)
-    m_transfers[i].m_key_image_request = false;
-  for (size_t i = 0; i < outputs.second.size(); ++i)
+  if (offset + output_array.size() > m_transfers.size())
+    m_transfers.resize(offset + output_array.size());
+  else if (num_outputs < m_transfers.size())
+    m_transfers.resize(num_outputs);
+
+  for (size_t i = 0; i < output_array.size(); ++i)
   {
-    transfer_details td = outputs.second[i];
+    transfer_details td = output_array[i];
 
     // skip those we've already imported, or which have different data
     if (i + offset < original_size)
@@ -13228,23 +13250,34 @@ process:
   return m_transfers.size();
 }
 //----------------------------------------------------------------------------------------------------
-size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> &outputs)
+size_t wallet2::import_outputs(const std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> &outputs)
 {
   PERF_TIMER(import_outputs);
 
   THROW_WALLET_EXCEPTION_IF(!m_offline, error::wallet_internal_error, "Hot wallets cannot import outputs");
 
-  THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error,
+  // we can now import piecemeal
+  const size_t offset = std::get<0>(outputs);
+  const size_t num_outputs = std::get<1>(outputs);
+  const std::vector<tools::wallet2::exported_transfer_details> &output_array = std::get<2>(outputs);
+
+  THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error,
       "Imported outputs omit more outputs that we know of. Try using export_outputs all.");
 
-  const size_t offset = outputs.first;
+  THROW_WALLET_EXCEPTION_IF(offset >= num_outputs, error::wallet_internal_error,
+      "Offset is larger than total outputs");
+  THROW_WALLET_EXCEPTION_IF(output_array.size() > num_outputs - offset, error::wallet_internal_error,
+      "Offset is larger than total outputs");
+
   const size_t original_size = m_transfers.size();
-  m_transfers.resize(offset + outputs.second.size());
-  for (size_t i = 0; i < offset; ++i)
-    m_transfers[i].m_key_image_request = false;
-  for (size_t i = 0; i < outputs.second.size(); ++i)
+  if (offset + output_array.size() > m_transfers.size())
+    m_transfers.resize(offset + output_array.size());
+  else if (num_outputs < m_transfers.size())
+    m_transfers.resize(num_outputs);
+
+  for (size_t i = 0; i < output_array.size(); ++i)
   {
-    exported_transfer_details etd = outputs.second[i];
+    exported_transfer_details etd = output_array[i];
     transfer_details &td = m_transfers[i + offset];
 
     // setup td with "cheap" loaded data
@@ -13358,7 +13391,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
   {
     std::string body(data, headerlen);
 
-    std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> new_outputs;
+    std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> new_outputs;
     try
     {
       binary_archive<false> ar{epee::strspan<std::uint8_t>(body)};
@@ -13368,9 +13401,9 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
     }
     catch (...) {}
     if (!loaded)
-      new_outputs.second.clear();
+      std::get<2>(new_outputs).clear();
 
-    std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs;
+    std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::transfer_details>> outputs;
     if (!loaded) try
     {
       binary_archive<false> ar{epee::strspan<std::uint8_t>(body)};
@@ -13395,11 +13428,12 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
 
     if (!loaded)
     {
-      outputs.first = 0;
-      outputs.second = {};
+      std::get<0>(outputs) = 0;
+      std::get<1>(outputs) = 0;
+      std::get<2>(outputs) = {};
     }
 
-    imported_outputs = !new_outputs.second.empty() ? import_outputs(new_outputs) : !outputs.second.empty() ? import_outputs(outputs) : 0;
+    imported_outputs = !std::get<2>(new_outputs).empty() ? import_outputs(new_outputs) : !std::get<2>(outputs).empty() ? import_outputs(outputs) : 0;
   }
   catch (const std::exception &e)
   {
diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h
index 06d4e636e..e684505f6 100644
--- a/src/wallet/wallet2.h
+++ b/src/wallet/wallet2.h
@@ -63,6 +63,7 @@
 #include "serialization/crypto.h"
 #include "serialization/string.h"
 #include "serialization/pair.h"
+#include "serialization/tuple.h"
 #include "serialization/containers.h"
 
 #include "wallet_errors.h"
@@ -673,16 +674,32 @@ private:
     struct unsigned_tx_set
     {
       std::vector<tx_construction_data> txes;
-      std::pair<size_t, wallet2::transfer_container> transfers;
-      std::pair<size_t, std::vector<wallet2::exported_transfer_details>> new_transfers;
+      std::tuple<uint64_t, uint64_t, wallet2::transfer_container> transfers;
+      std::tuple<uint64_t, uint64_t, std::vector<wallet2::exported_transfer_details>> new_transfers;
 
       BEGIN_SERIALIZE_OBJECT()
-        VERSION_FIELD(1)
+        VERSION_FIELD(2)
         FIELD(txes)
-        if (version >= 1)
-          FIELD(new_transfers)
-        else
-          FIELD(transfers)
+        if (version == 0)
+        {
+          std::pair<size_t, wallet2::transfer_container> v0_transfers;
+          FIELD(v0_transfers);
+          std::get<0>(transfers) = std::get<0>(v0_transfers);
+          std::get<1>(transfers) = std::get<0>(v0_transfers) + std::get<1>(v0_transfers).size();
+          std::get<2>(transfers) = std::get<1>(v0_transfers);
+          return true;
+        }
+        if (version == 1)
+        {
+          std::pair<size_t, std::vector<wallet2::exported_transfer_details>> v1_transfers;
+          FIELD(v1_transfers);
+          std::get<0>(new_transfers) = std::get<0>(v1_transfers);
+          std::get<1>(new_transfers) = std::get<0>(v1_transfers) + std::get<1>(v1_transfers).size();
+          std::get<2>(new_transfers) = std::get<1>(v1_transfers);
+          return true;
+        }
+
+        FIELD(new_transfers)
       END_SERIALIZE()
     };
 
@@ -1453,10 +1470,10 @@ private:
     bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const;
 
     // Import/Export wallet data
-    std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> export_outputs(bool all = false) const;
-    std::string export_outputs_to_str(bool all = false) const;
-    size_t import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::exported_transfer_details>> &outputs);
-    size_t import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs);
+    std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> export_outputs(bool all = false, uint32_t start = 0, uint32_t count = 0xffffffff) const;
+    std::string export_outputs_to_str(bool all = false, uint32_t start = 0, uint32_t count = 0) const;
+    size_t import_outputs(const std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::exported_transfer_details>> &outputs);
+    size_t import_outputs(const std::tuple<uint64_t, uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs);
     size_t import_outputs_from_str(const std::string &outputs_st);
     payment_container export_payments() const;
     void import_payments(const payment_container &payments);
@@ -1904,7 +1921,7 @@ BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 8)
 BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6)
 BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 18)
 BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0)
-BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0)
+BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 1)
 BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1)
 BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 4)
 BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3)
@@ -1914,6 +1931,17 @@ namespace boost
 {
   namespace serialization
   {
+    template<class Archive, class F, class S, class T>
+    inline void serialize(
+        Archive & ar,
+        std::tuple<F, S, T> & t,
+        const unsigned int /* file_version */
+    ){
+        ar & boost::serialization::make_nvp("f", std::get<0>(t));
+        ar & boost::serialization::make_nvp("s", std::get<1>(t));
+        ar & boost::serialization::make_nvp("t", std::get<2>(t));
+    }
+
     template <class Archive>
     inline typename std::enable_if<!Archive::is_loading::value, void>::type initialize_transfer_details(Archive &a, tools::wallet2::transfer_details &x, const boost::serialization::version_type ver)
     {
@@ -2274,7 +2302,17 @@ namespace boost
     inline void serialize(Archive &a, tools::wallet2::unsigned_tx_set &x, const boost::serialization::version_type ver)
     {
       a & x.txes;
-      a & x.transfers;
+      if (ver == 0)
+      {
+        // load old version
+        std::pair<size_t, tools::wallet2::transfer_container> old_transfers;
+        a & old_transfers;
+        std::get<0>(x.transfers) = std::get<0>(old_transfers);
+        std::get<1>(x.transfers) = std::get<0>(old_transfers) + std::get<1>(old_transfers).size();
+        std::get<2>(x.transfers) = std::get<1>(old_transfers);
+        return;
+      }
+      throw std::runtime_error("Boost serialization not supported for newest unsigned_tx_set");
     }
 
     template <class Archive>
diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp
index 7ec5fc7a1..1f0a1371f 100644
--- a/src/wallet/wallet_rpc_server.cpp
+++ b/src/wallet/wallet_rpc_server.cpp
@@ -2792,7 +2792,7 @@ namespace tools
 
     try
     {
-      res.outputs_data_hex = epee::string_tools::buff_to_hex_nodelimer(m_wallet->export_outputs_to_str(req.all));
+      res.outputs_data_hex = epee::string_tools::buff_to_hex_nodelimer(m_wallet->export_outputs_to_str(req.all, req.start, req.count));
     }
     catch (const std::exception &e)
     {
diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h
index ecfc8e673..2cca323ee 100644
--- a/src/wallet/wallet_rpc_server_commands_defs.h
+++ b/src/wallet/wallet_rpc_server_commands_defs.h
@@ -1785,9 +1785,13 @@ namespace wallet_rpc
     struct request_t
     {
       bool all;
+      uint32_t start;
+      uint32_t count;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(all)
+        KV_SERIALIZE_OPT(start, 0u)
+        KV_SERIALIZE_OPT(count, 0xffffffffu)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;