replace most boost serialization with existing monero serialization

This reduces the attack surface for data that can come from malicious sources (exported output and key images, multisig transactions...) since the monero serialization is already exposed to the outside, and the boost lib we were using had a few known crashers. For interoperability, a new load-deprecated-formats wallet setting is added (off by default). This allows loading boost format data if there is no alternative. It will likely go at some point, along with the ability to load those. Notably, the peer lists file still uses the boost serialization code, as the data it stores is define in epee, while the new serialization code is in monero, and migrating it was fairly hairy. Since this file is local and not obtained from anyone else, the marginal risk is minimal, but it could be migrated later if needed. Some tests and tools also do, this will stay as is for now.
author: moneromooo-monero <moneromooo-monero@users.noreply.github.com> 2020-06-24 23:26:58 +0000
committer: moneromooo-monero <moneromooo-monero@users.noreply.github.com> 2020-08-17 16:23:58 +0000
commit: 7175dcb1078abbdaa130a8c5f5fd2b93fa7b3086 (patch)
tree: 640b83c4101dc992151630a76a13597913d2dd08 /src
parent: Merge pull request #6736 (diff)
download: monero-7175dcb1078abbdaa130a8c5f5fd2b93fa7b3086.tar.xz
24 files changed, 826 insertions, 379 deletions
diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp
index a847f6a9d..a8197483f 100644
--- a/src/blockchain_utilities/blockchain_blackball.cpp
+++ b/src/blockchain_utilities/blockchain_blackball.cpp
@@ -28,8 +28,6 @@
 
 #include <boost/range/adaptor/transformed.hpp>
 #include <boost/algorithm/string.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-#include <boost/archive/portable_binary_oarchive.hpp>
 #include "common/unordered_containers_boost_serialization.h"
 #include "common/command_line.h"
 #include "common/varint.h"
diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h
index bc6a378f2..c70ae1df1 100644
--- a/src/cryptonote_basic/cryptonote_basic.h
+++ b/src/cryptonote_basic/cryptonote_basic.h
@@ -37,7 +37,7 @@
 #include <sstream>
 #include <atomic>
 #include "serialization/variant.h"
-#include "serialization/vector.h"
+#include "serialization/containers.h"
 #include "serialization/binary_archive.h"
 #include "serialization/json_archive.h"
 #include "serialization/debug_archive.h"
diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp
index 541393fa9..2600854a9 100644
--- a/src/cryptonote_basic/cryptonote_basic_impl.cpp
+++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp
@@ -34,7 +34,6 @@ using namespace epee;
 #include "cryptonote_basic_impl.h"
 #include "string_tools.h"
 #include "serialization/binary_utils.h"
-#include "serialization/container.h"
 #include "cryptonote_format_utils.h"
 #include "cryptonote_config.h"
 #include "misc_language.h"
diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h
index 6f89cc7ae..b3d39a616 100644
--- a/src/cryptonote_basic/cryptonote_boost_serialization.h
+++ b/src/cryptonote_basic/cryptonote_boost_serialization.h
@@ -36,7 +36,6 @@
 #include <boost/serialization/set.hpp>
 #include <boost/serialization/map.hpp>
 #include <boost/serialization/is_bitwise_serializable.hpp>
-#include <boost/archive/binary_iarchive.hpp>
 #include <boost/archive/portable_binary_iarchive.hpp>
 #include <boost/archive/portable_binary_oarchive.hpp>
 #include "cryptonote_basic.h"
diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h
index bbc165cfa..b439dc47e 100644
--- a/src/p2p/p2p_protocol_defs.h
+++ b/src/p2p/p2p_protocol_defs.h
@@ -39,6 +39,7 @@
 #include "misc_language.h"
 #include "string_tools.h"
 #include "time_helper.h"
+#include "serialization/serialization.h"
 #include "cryptonote_config.h"
 
 namespace nodetool
@@ -84,6 +85,15 @@ namespace nodetool
       KV_SERIALIZE_OPT(rpc_port, (uint16_t)0)
       KV_SERIALIZE_OPT(rpc_credits_per_hash, (uint32_t)0)
     END_KV_SERIALIZE_MAP()
+
+    BEGIN_SERIALIZE()
+      FIELD(adr)
+      FIELD(id)
+      VARINT_FIELD(last_seen)
+      VARINT_FIELD(pruning_seed)
+      VARINT_FIELD(rpc_port)
+      VARINT_FIELD(rpc_credits_per_hash)
+    END_SERIALIZE()
   };
   typedef peerlist_entry_base<epee::net_utils::network_address> peerlist_entry;
 
@@ -99,6 +109,12 @@ namespace nodetool
       KV_SERIALIZE(id)
       KV_SERIALIZE(first_seen)
     END_KV_SERIALIZE_MAP()
+
+    BEGIN_SERIALIZE()
+      FIELD(adr)
+      FIELD(id)
+      VARINT_FIELD(first_seen)
+    END_SERIALIZE()
   };
   typedef anchor_peerlist_entry_base<epee::net_utils::network_address> anchor_peerlist_entry;
 
@@ -114,6 +130,12 @@ namespace nodetool
       KV_SERIALIZE(id)
       KV_SERIALIZE(is_income)
     END_KV_SERIALIZE_MAP()
+
+    BEGIN_SERIALIZE()
+      FIELD(adr)
+      FIELD(id)
+      FIELD(is_income)
+    END_SERIALIZE()
   };
   typedef connection_entry_base<epee::net_utils::network_address> connection_entry;
 
diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h
index 9b7f26a02..ce11981ad 100644
--- a/src/ringct/rctTypes.h
+++ b/src/ringct/rctTypes.h
@@ -49,7 +49,7 @@ extern "C" {
 #include "hex.h"
 #include "span.h"
 #include "memwipe.h"
-#include "serialization/vector.h"
+#include "serialization/containers.h"
 #include "serialization/debug_archive.h"
 #include "serialization/binary_archive.h"
 #include "serialization/json_archive.h"
@@ -239,6 +239,12 @@ namespace rct {
     struct RCTConfig {
       RangeProofType range_proof_type;
       int bp_version;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        VARINT_FIELD(range_proof_type)
+        VARINT_FIELD(bp_version)
+      END_SERIALIZE()
     };
     struct rctSigBase {
         uint8_t type;
@@ -317,6 +323,16 @@ namespace rct {
           ar.end_array();
           return ar.stream().good();
         }
+
+        BEGIN_SERIALIZE_OBJECT()
+          FIELD(type)
+          FIELD(message)
+          FIELD(mixRing)
+          FIELD(pseudoOuts)
+          FIELD(ecdhInfo)
+          FIELD(outPk)
+          VARINT_FIELD(txnFee)
+        END_SERIALIZE()
     };
     struct rctSigPrunable {
         std::vector<rangeSig> rangeSigs;
@@ -436,6 +452,12 @@ namespace rct {
           return ar.stream().good();
         }
 
+        BEGIN_SERIALIZE_OBJECT()
+          FIELD(rangeSigs)
+          FIELD(bulletproofs)
+          FIELD(MGs)
+          FIELD(pseudoOuts)
+        END_SERIALIZE()
     };
     struct rctSig: public rctSigBase {
         rctSigPrunable p;
@@ -449,6 +471,11 @@ namespace rct {
         {
           return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts;
         }
+
+        BEGIN_SERIALIZE_OBJECT()
+          FIELDS((rctSigBase&)*this)
+          FIELD(p)
+        END_SERIALIZE()
     };
 
     //other basepoint H = toPoint(cn_fast_hash(G)), G the basepoint
diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp
index edc8f0dda..176f11fa3 100644
--- a/src/rpc/rpc_payment.cpp
+++ b/src/rpc/rpc_payment.cpp
@@ -27,14 +27,12 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <boost/archive/portable_binary_iarchive.hpp>
-#include <boost/archive/portable_binary_oarchive.hpp>
 #include "cryptonote_config.h"
 #include "include_base_utils.h"
 #include "string_tools.h"
 #include "file_io_utils.h"
 #include "int-util.h"
 #include "common/util.h"
-#include "serialization/crypto.h"
 #include "common/unordered_containers_boost_serialization.h"
 #include "cryptonote_basic/cryptonote_boost_serialization.h"
 #include "cryptonote_basic/cryptonote_format_utils.h"
@@ -296,14 +294,28 @@ namespace cryptonote
     data.open(state_file_path, std::ios_base::binary | std::ios_base::in);
     if (!data.fail())
     {
+      bool loaded = false;
       try
       {
-        boost::archive::portable_binary_iarchive a(data);
-        a >> *this;
+        binary_archive<false> ar(data);
+        if (::serialization::serialize(ar, *this))
+          if (::serialization::check_stream_state(ar))
+            loaded = true;
       }
-      catch (const std::exception &e)
+      catch (...) {}
+      if (!loaded)
       {
-        MERROR("Failed to load RPC payments file: " << e.what());
+        try
+        {
+          boost::archive::portable_binary_iarchive a(data);
+          a >> *this;
+          loaded = true;
+        }
+        catch (...) {}
+      }
+      if (!loaded)
+      {
+        MERROR("Failed to load RPC payments file");
         m_client_info.clear();
       }
     }
@@ -344,8 +356,9 @@ namespace cryptonote
       MWARNING("Failed to save RPC payments to file " << state_file_path);
       return false;
     };
-    boost::archive::portable_binary_oarchive a(data);
-    a << *this;
+    binary_archive<true> ar(data);
+    if (!::serialization::serialize(ar, *const_cast<rpc_payment*>(this)))
+      return false;
     return true;
     CATCH_ENTRY_L0("rpc_payment::store", false);
   }
diff --git a/src/rpc/rpc_payment.h b/src/rpc/rpc_payment.h
index dcd43f8d5..fdf1f953f 100644
--- a/src/rpc/rpc_payment.h
+++ b/src/rpc/rpc_payment.h
@@ -31,10 +31,17 @@
 #include <string>
 #include <unordered_set>
 #include <unordered_map>
+#include <map>
 #include <boost/thread/mutex.hpp>
 #include <boost/serialization/version.hpp>
 #include "cryptonote_basic/blobdatatype.h"
 #include "cryptonote_basic/cryptonote_basic.h"
+#include <boost/serialization/list.hpp>
+#include <boost/serialization/vector.hpp>
+#include "serialization/crypto.h"
+#include "serialization/string.h"
+#include "serialization/pair.h"
+#include "serialization/containers.h"
 
 namespace cryptonote
 {
@@ -96,6 +103,33 @@ namespace cryptonote
         a & nonces_bad;
         a & nonces_dupe;
       }
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(block)
+        FIELD(previous_block)
+        FIELD(hashing_blob)
+        FIELD(previous_hashing_blob)
+        VARINT_FIELD(seed_height)
+        VARINT_FIELD(previous_seed_height)
+        FIELD(seed_hash)
+        FIELD(previous_seed_hash)
+        VARINT_FIELD(cookie)
+        FIELD(top)
+        FIELD(previous_top)
+        VARINT_FIELD(credits)
+        FIELD(payments)
+        FIELD(previous_payments)
+        FIELD(update_time)
+        FIELD(last_request_timestamp)
+        FIELD(block_template_update_time)
+        VARINT_FIELD(credits_total)
+        VARINT_FIELD(credits_used)
+        VARINT_FIELD(nonces_good)
+        VARINT_FIELD(nonces_stale)
+        VARINT_FIELD(nonces_bad)
+        VARINT_FIELD(nonces_dupe)
+      END_SERIALIZE()
     };
 
   public:
@@ -114,8 +148,8 @@ namespace cryptonote
     template <class t_archive>
     inline void serialize(t_archive &a, const unsigned int ver)
     {
-      a & m_client_info;
-      a & m_hashrate;
+      a & m_client_info.parent();
+      a & m_hashrate.parent();
       a & m_credits_total;
       a & m_credits_used;
       a & m_nonces_good;
@@ -124,6 +158,18 @@ namespace cryptonote
       a & m_nonces_dupe;
     }
 
+    BEGIN_SERIALIZE_OBJECT()
+      VERSION_FIELD(0)
+      FIELD(m_client_info)
+      FIELD(m_hashrate)
+      VARINT_FIELD(m_credits_total)
+      VARINT_FIELD(m_credits_used)
+      VARINT_FIELD(m_nonces_good)
+      VARINT_FIELD(m_nonces_stale)
+      VARINT_FIELD(m_nonces_bad)
+      VARINT_FIELD(m_nonces_dupe)
+    END_SERIALIZE()
+
     bool load(std::string directory);
     bool store(const std::string &directory = std::string()) const;
 
@@ -131,9 +177,9 @@ namespace cryptonote
     cryptonote::account_public_address m_address;
     uint64_t m_diff;
     uint64_t m_credits_per_hash_found;
-    std::unordered_map<crypto::public_key, client_info> m_client_info;
+    serializable_unordered_map<crypto::public_key, client_info> m_client_info;
     std::string m_directory;
-    std::map<uint64_t, uint64_t> m_hashrate;
+    serializable_map<uint64_t, uint64_t> m_hashrate;
     uint64_t m_credits_total;
     uint64_t m_credits_used;
     uint64_t m_nonces_good;
@@ -143,6 +189,3 @@ namespace cryptonote
     mutable boost::mutex mutex;
   };
 }
-
-BOOST_CLASS_VERSION(cryptonote::rpc_payment, 0);
-BOOST_CLASS_VERSION(cryptonote::rpc_payment::client_info, 0);
diff --git a/src/serialization/container.h b/src/serialization/container.h
index 4bf47ecfa..d5e75bb4f 100644
--- a/src/serialization/container.h
+++ b/src/serialization/container.h
@@ -28,10 +28,6 @@
 // 
 // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
 
-#pragma once
-
-#include "serialization.h"
-
 namespace serialization
 {
   namespace detail
@@ -103,7 +99,7 @@ bool do_serialize_container(Archive<true> &ar, C &v)
       return false;
     if (i != v.begin())
       ar.delimit_array();
-    if(!::serialization::detail::serialize_container_element(ar, const_cast<typename C::value_type&>(*i)))
+    if(!::serialization::detail::serialize_container_element(ar, (typename C::value_type&)*i))
       return false;
     if (!ar.stream().good())
       return false;
diff --git a/src/serialization/containers.h b/src/serialization/containers.h
new file mode 100644
index 000000000..bc4a89527
--- /dev/null
+++ b/src/serialization/containers.h
@@ -0,0 +1,128 @@
+// Copyright (c) 2014-2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once
+
+#include <vector>
+#include <deque>
+#include <unordered_map>
+#include <map>
+#include <unordered_set>
+#include <set>
+#include "serialization.h"
+
+template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::vector<T> &v);
+template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::vector<T> &v);
+
+template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::deque<T> &v);
+template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::deque<T> &v);
+
+template<typename K, typename V>
+class serializable_unordered_map: public std::unordered_map<K, V>
+{
+public:
+  typedef typename std::pair<K, V> value_type;
+  typename std::unordered_map<K, V> &parent() { return *this; }
+};
+
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_map<K, V> &v);
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_map<K, V> &v);
+
+template<typename K, typename V>
+class serializable_map: public std::map<K, V>
+{
+public:
+  typedef typename std::pair<K, V> value_type;
+  typename std::map<K, V> &parent() { return *this; }
+};
+
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_map<K, V> &v);
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_map<K, V> &v);
+
+template<typename K, typename V>
+class serializable_unordered_multimap: public std::unordered_multimap<K, V>
+{
+public:
+  typedef typename std::pair<K, V> value_type;
+  typename std::unordered_multimap<K, V> &parent() { return *this; }
+};
+
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_multimap<K, V> &v);
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_multimap<K, V> &v);
+
+template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v);
+template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v);
+
+template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::set<T> &v);
+template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::set<T> &v);
+
+namespace serialization
+{
+  namespace detail
+  {
+    template <typename T> void do_reserve(std::vector<T> &c, size_t N) { c.reserve(N); }
+    template <typename T> void do_add(std::vector<T> &c, T &&e) { c.emplace_back(std::forward<T>(e)); }
+
+    template <typename T> void do_add(std::deque<T> &c, T &&e) { c.emplace_back(std::forward<T>(e)); }
+
+    template <typename K, typename V> void do_add(serializable_unordered_map<K, V> &c, std::pair<K, V> &&e) { c.insert(std::forward<std::pair<K, V>>(e)); }
+
+    template <typename K, typename V> void do_add(serializable_map<K, V> &c, std::pair<K, V> &&e) { c.insert(std::forward<std::pair<K, V>>(e)); }
+
+    template <typename K, typename V> void do_add(serializable_unordered_multimap<K, V> &c, std::pair<K, V> &&e) { c.insert(std::forward<std::pair<K, V>>(e)); }
+
+    template <typename T> void do_add(std::unordered_set<T> &c, T &&e) { c.insert(std::forward<T>(e)); }
+
+    template <typename T> void do_add(std::set<T> &c, T &&e) { c.insert(std::forward<T>(e)); }
+  }
+}
+
+#include "container.h"
+
+template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); }
+template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); }
+
+template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); }
+template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); }
+
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_map<K, V> &v) { return do_serialize_container(ar, v); }
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_map<K, V> &v) { return do_serialize_container(ar, v); }
+
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_map<K, V> &v) { return do_serialize_container(ar, v); }
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_map<K, V> &v) { return do_serialize_container(ar, v); }
+
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_multimap<K, V> &v) { return do_serialize_container(ar, v); }
+template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_multimap<K, V> &v) { return do_serialize_container(ar, v); }
+
+template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); }
+template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); }
+
+template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::set<T> &v) { return do_serialize_container(ar, v); }
+template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::set<T> &v) { return do_serialize_container(ar, v); }
diff --git a/src/serialization/deque.h b/src/serialization/deque.h
deleted file mode 100644
index b029ed430..000000000
--- a/src/serialization/deque.h
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright (c) 2014-2020, The Monero Project
-// 
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without modification, are
-// permitted provided that the following conditions are met:
-// 
-// 1. Redistributions of source code must retain the above copyright notice, this list of
-//    conditions and the following disclaimer.
-// 
-// 2. Redistributions in binary form must reproduce the above copyright notice, this list
-//    of conditions and the following disclaimer in the documentation and/or other
-//    materials provided with the distribution.
-// 
-// 3. Neither the name of the copyright holder nor the names of its contributors may be
-//    used to endorse or promote products derived from this software without specific
-//    prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
-// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
-
-#pragma once
-
-#include <deque>
-
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<false> &ar, std::deque<T> &v);
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<true> &ar, std::deque<T> &v);
-
-namespace serialization
-{
-  namespace detail
-  {
-    template <typename T>
-    void do_reserve(std::deque<T> &c, size_t N)
-    {
-      c.reserve(N);
-    }
-
-    template <typename T>
-    void do_add(std::deque<T> &c, T &&e)
-    {
-      c.emplace_back(std::move(e));
-    }
-  }
-}
-
-#include "serialization.h"
-
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<false> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); }
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<true> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); }
-
diff --git a/src/serialization/list.h b/src/serialization/list.h
index 139269b40..411e7800d 100644
--- a/src/serialization/list.h
+++ b/src/serialization/list.h
@@ -44,7 +44,7 @@ namespace serialization
     template <typename T>
     void do_add(std::list<T> &c, T &&e)
     {
-      c.emplace_back(std::move(e));
+      c.emplace_back(std::forward<T>(e));
     }
   }
 }
diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h
index d0f9ee0f7..3ebeed171 100644
--- a/src/serialization/serialization.h
+++ b/src/serialization/serialization.h
@@ -48,6 +48,7 @@
 #include <string>
 #include <boost/type_traits/is_integral.hpp>
 #include <boost/type_traits/integral_constant.hpp>
+#include <boost/mpl/bool.hpp>
 
 /*! \struct is_blob_type 
  *
@@ -278,6 +279,27 @@ inline bool do_serialize(Archive &ar, bool &v)
     if (!ar.stream().good()) return false;	\
   } while(0);
 
+/*! \macro MAGIC_FIELD(m)
+ */
+#define MAGIC_FIELD(m)				\
+  std::string magic = m;			\
+  do {						\
+    ar.tag("magic");				\
+    ar.serialize_blob((void*)magic.data(), magic.size()); \
+    if (!ar.stream().good()) return false;	\
+    if (magic != m) return false;		\
+  } while(0);
+
+/*! \macro VERSION_FIELD(v)
+ */
+#define VERSION_FIELD(v)			\
+  uint32_t version = v;				\
+  do {						\
+    ar.tag("version");				\
+    ar.serialize_varint(version);		\
+    if (!ar.stream().good()) return false;	\
+  } while(0);
+
 
 namespace serialization {
   /*! \namespace detail
diff --git a/src/serialization/set.h b/src/serialization/set.h
deleted file mode 100644
index 157c002f7..000000000
--- a/src/serialization/set.h
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright (c) 2014-2020, The Monero Project
-// 
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without modification, are
-// permitted provided that the following conditions are met:
-// 
-// 1. Redistributions of source code must retain the above copyright notice, this list of
-//    conditions and the following disclaimer.
-// 
-// 2. Redistributions in binary form must reproduce the above copyright notice, this list
-//    of conditions and the following disclaimer in the documentation and/or other
-//    materials provided with the distribution.
-// 
-// 3. Neither the name of the copyright holder nor the names of its contributors may be
-//    used to endorse or promote products derived from this software without specific
-//    prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
-// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
-
-#pragma once
-
-#include <set>
-
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<false> &ar, std::set<T> &v);
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<true> &ar, std::set<T> &v);
-
-namespace serialization
-{
-  namespace detail
-  {
-    template <typename T>
-    void do_add(std::set<T> &c, T &&e)
-    {
-      c.insert(std::move(e));
-    }
-  }
-}
-
-#include "serialization.h"
-
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<false> &ar, std::set<T> &v) { return do_serialize_container(ar, v); }
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<true> &ar, std::set<T> &v) { return do_serialize_container(ar, v); }
-
diff --git a/src/serialization/unordered_set.h b/src/serialization/unordered_set.h
deleted file mode 100644
index 24aa09437..000000000
--- a/src/serialization/unordered_set.h
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright (c) 2014-2020, The Monero Project
-// 
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without modification, are
-// permitted provided that the following conditions are met:
-// 
-// 1. Redistributions of source code must retain the above copyright notice, this list of
-//    conditions and the following disclaimer.
-// 
-// 2. Redistributions in binary form must reproduce the above copyright notice, this list
-//    of conditions and the following disclaimer in the documentation and/or other
-//    materials provided with the distribution.
-// 
-// 3. Neither the name of the copyright holder nor the names of its contributors may be
-//    used to endorse or promote products derived from this software without specific
-//    prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
-// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
-
-#pragma once
-
-#include <set>
-
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v);
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v);
-
-namespace serialization
-{
-  namespace detail
-  {
-    template <typename T>
-    void do_add(std::unordered_set<T> &c, T &&e)
-    {
-      c.insert(std::move(e));
-    }
-  }
-}
-
-#include "serialization.h"
-
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); }
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); }
-
diff --git a/src/serialization/vector.h b/src/serialization/vector.h
deleted file mode 100644
index 1fca1d485..000000000
--- a/src/serialization/vector.h
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright (c) 2014-2020, The Monero Project
-// 
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without modification, are
-// permitted provided that the following conditions are met:
-// 
-// 1. Redistributions of source code must retain the above copyright notice, this list of
-//    conditions and the following disclaimer.
-// 
-// 2. Redistributions in binary form must reproduce the above copyright notice, this list
-//    of conditions and the following disclaimer in the documentation and/or other
-//    materials provided with the distribution.
-// 
-// 3. Neither the name of the copyright holder nor the names of its contributors may be
-//    used to endorse or promote products derived from this software without specific
-//    prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
-// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
-
-#pragma once
-
-#include <vector>
-#include "serialization.h"
-
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<false> &ar, std::vector<T> &v);
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<true> &ar, std::vector<T> &v);
-
-namespace serialization
-{
-  namespace detail
-  {
-    template <typename T>
-    void do_reserve(std::vector<T> &c, size_t N)
-    {
-      c.reserve(N);
-    }
-
-    template <typename T>
-    void do_add(std::vector<T> &c, T &&e)
-    {
-      c.emplace_back(std::move(e));
-    }
-  }
-}
-
-#include "container.h"
-
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<false> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); }
-template <template <bool> class Archive, class T>
-bool do_serialize(Archive<true> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); }
-
diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp
index 69c684097..bbf303ea5 100644
--- a/src/simplewallet/simplewallet.cpp
+++ b/src/simplewallet/simplewallet.cpp
@@ -3100,6 +3100,28 @@ bool simple_wallet::set_export_format(const std::vector<std::string> &args/* = s
   return true;
 }
 
+bool simple_wallet::set_load_deprecated_formats(const std::vector<std::string> &args/* = std::vector<std::string()*/)
+{
+  if (args.size() < 2)
+  {
+    fail_msg_writer() << tr("Value not specified");
+    return true;
+  }
+
+  const auto pwd_container = get_and_verify_password();
+  if (pwd_container)
+  {
+    parse_bool_and_use(args[1], [&](bool r) {
+      m_wallet->load_deprecated_formats(r);
+      m_wallet->rewrite(m_wallet_file, pwd_container->password());
+
+      if (r)
+        message_writer() << tr("Warning: deprecated formats use boost serialization, which has buffer overflows and crashers. Only load deprecated formats from sources you trust.");
+    });
+  }
+  return true;
+}
+
 bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   if(args.empty())
@@ -3789,6 +3811,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     success_msg_writer() << "persistent-rpc-client-id = " << m_wallet->persistent_rpc_client_id();
     success_msg_writer() << "auto-mine-for-rpc-payment-threshold = " << m_wallet->auto_mine_for_rpc_payment_threshold();
     success_msg_writer() << "credits-target = " << m_wallet->credits_target();
+    success_msg_writer() << "load-deprecated-formats = " << m_wallet->load_deprecated_formats();
     return true;
   }
   else
@@ -3850,6 +3873,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     CHECK_SIMPLE_VARIABLE("setup-background-mining", set_setup_background_mining, tr("1/yes or 0/no"));
     CHECK_SIMPLE_VARIABLE("device-name", set_device_name, tr("<device_name[:device_spec]>"));
     CHECK_SIMPLE_VARIABLE("export-format", set_export_format, tr("\"binary\" or \"ascii\""));
+    CHECK_SIMPLE_VARIABLE("load-deprecated-formats", set_load_deprecated_formats, tr("0 or 1"));
     CHECK_SIMPLE_VARIABLE("persistent-rpc-client-id", set_persistent_rpc_client_id, tr("0 or 1"));
     CHECK_SIMPLE_VARIABLE("auto-mine-for-rpc-payment-threshold", set_auto_mine_for_rpc_payment_threshold, tr("floating point >= 0"));
     CHECK_SIMPLE_VARIABLE("credits-target", set_credits_target, tr("unsigned integer"));
diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h
index 60aa6c4f6..5846fe056 100644
--- a/src/simplewallet/simplewallet.h
+++ b/src/simplewallet/simplewallet.h
@@ -151,6 +151,7 @@ namespace cryptonote
     bool set_setup_background_mining(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_device_name(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_export_format(const std::vector<std::string> &args = std::vector<std::string>());
+    bool set_load_deprecated_formats(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_persistent_rpc_client_id(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_auto_mine_for_rpc_payment_threshold(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_credits_target(const std::vector<std::string> &args = std::vector<std::string>());
diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp
index fb07b42f0..303b576c7 100644
--- a/src/wallet/message_store.cpp
+++ b/src/wallet/message_store.cpp
@@ -27,7 +27,6 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "message_store.h"
-#include <boost/archive/portable_binary_oarchive.hpp>
 #include <boost/archive/portable_binary_iarchive.hpp>
 #include <boost/format.hpp>
 #include <boost/algorithm/string.hpp>
@@ -182,8 +181,8 @@ bool message_store::signer_labels_complete() const
 void message_store::get_signer_config(std::string &signer_config)
 {
   std::stringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
-  ar << m_signers;
+  binary_archive<true> ar(oss);
+  THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, m_signers), tools::error::wallet_internal_error, "Failed to serialize signer config");
   signer_config = oss.str();
 }
 
@@ -194,8 +193,8 @@ void message_store::unpack_signer_config(const multisig_wallet_state &state, con
   {
     std::stringstream iss;
     iss << signer_config;
-    boost::archive::portable_binary_iarchive ar(iss);
-    ar >> signers;
+    binary_archive<false> ar(iss);
+    THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, signers), tools::error::wallet_internal_error, "Failed to serialize signer config");
   }
   catch (...)
   {
@@ -364,8 +363,8 @@ size_t message_store::add_auto_config_data_message(const multisig_wallet_state &
   data.monero_address = me.monero_address;
 
   std::stringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
-  ar << data;
+  binary_archive<true> ar(oss);
+  THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, data), tools::error::wallet_internal_error, "Failed to serialize auto config data");
 
   return add_message(state, 0, message_type::auto_config_data, message_direction::out, oss.str());
 }
@@ -384,8 +383,8 @@ void message_store::process_auto_config_data_message(uint32_t id)
   {
     std::stringstream iss;
     iss << m.content;
-    boost::archive::portable_binary_iarchive ar(iss);
-    ar >> data;
+    binary_archive<false> ar(iss);
+    THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, data), tools::error::wallet_internal_error, "Failed to serialize auto config data");
   }
   catch (...)
   {
@@ -745,8 +744,8 @@ std::string message_store::get_sanitized_text(const std::string &text, size_t ma
 void message_store::write_to_file(const multisig_wallet_state &state, const std::string &filename)
 {
   std::stringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
-  ar << *this;
+  binary_archive<true> ar(oss);
+  THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, *this), tools::error::wallet_internal_error, "Failed to serialize MMS state");
   std::string buf = oss.str();
 
   crypto::chacha_key key;
@@ -762,14 +761,14 @@ void message_store::write_to_file(const multisig_wallet_state &state, const std:
   write_file_data.encrypted_data = encrypted_data;
 
   std::stringstream file_oss;
-  boost::archive::portable_binary_oarchive file_ar(file_oss);
-  file_ar << write_file_data;
+  binary_archive<true> file_ar(file_oss);
+  THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(file_ar, write_file_data), tools::error::wallet_internal_error, "Failed to serialize MMS state");
 
   bool success = epee::file_io_utils::save_string_to_file(filename, file_oss.str());
   THROW_WALLET_EXCEPTION_IF(!success, tools::error::file_save_error, filename);
 }
 
-void message_store::read_from_file(const multisig_wallet_state &state, const std::string &filename)
+void message_store::read_from_file(const multisig_wallet_state &state, const std::string &filename, bool load_deprecated_formats)
 {
   boost::system::error_code ignored_ec;
   bool file_exists = boost::filesystem::exists(filename, ignored_ec);
@@ -785,17 +784,37 @@ void message_store::read_from_file(const multisig_wallet_state &state, const std
   bool success = epee::file_io_utils::load_file_to_string(filename, buf);
   THROW_WALLET_EXCEPTION_IF(!success, tools::error::file_read_error, filename);
 
+  bool loaded = false;
   file_data read_file_data;
   try
   {
     std::stringstream iss;
     iss << buf;
-    boost::archive::portable_binary_iarchive ar(iss);
-    ar >> read_file_data;
+    binary_archive<false> ar(iss);
+    if (::serialization::serialize(ar, read_file_data))
+      if (::serialization::check_stream_state(ar))
+        loaded = true;
   }
-  catch (const std::exception &e)
+  catch (...) {}
+  if (!loaded && load_deprecated_formats)
+  {
+    try
+    {
+      std::stringstream iss;
+      iss << buf;
+      boost::archive::portable_binary_iarchive ar(iss);
+      ar >> read_file_data;
+      loaded = true;
+    }
+    catch (const std::exception &e)
+    {
+      MERROR("MMS file " << filename << " has bad structure <iv,encrypted_data>: " << e.what());
+      THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename);
+    }
+  }
+  if (!loaded)
   {
-    MERROR("MMS file " << filename << " has bad structure <iv,encrypted_data>: " << e.what());
+    MERROR("MMS file " << filename << " has bad structure <iv,encrypted_data>");
     THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename);
   }
 
@@ -805,16 +824,36 @@ void message_store::read_from_file(const multisig_wallet_state &state, const std
   decrypted_data.resize(read_file_data.encrypted_data.size());
   crypto::chacha20(read_file_data.encrypted_data.data(), read_file_data.encrypted_data.size(), key, read_file_data.iv, &decrypted_data[0]);
 
+  loaded = false;
   try
   {
     std::stringstream iss;
     iss << decrypted_data;
-    boost::archive::portable_binary_iarchive ar(iss);
-    ar >> *this;
+    binary_archive<false> ar(iss);
+    if (::serialization::serialize(ar, *this))
+      if (::serialization::check_stream_state(ar))
+        loaded = true;
   }
-  catch (const std::exception &e)
+  catch(...) {}
+  if (!loaded && load_deprecated_formats)
+  {
+    try
+    {
+      std::stringstream iss;
+      iss << decrypted_data;
+      boost::archive::portable_binary_iarchive ar(iss);
+      ar >> *this;
+      loaded = true;
+    }
+    catch (const std::exception &e)
+    {
+      MERROR("MMS file " << filename << " has bad structure: " << e.what());
+      THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename);
+    }
+  }
+  if (!loaded)
   {
-    MERROR("MMS file " << filename << " has bad structure: " << e.what());
+    MERROR("MMS file " << filename << " has bad structure");
     THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename);
   }
 
diff --git a/src/wallet/message_store.h b/src/wallet/message_store.h
index 9055fd776..0f53587d4 100644
--- a/src/wallet/message_store.h
+++ b/src/wallet/message_store.h
@@ -44,6 +44,9 @@
 #include "common/command_line.h"
 #include "wipeable_string.h"
 #include "net/abstract_http_client.h"
+#include "serialization/crypto.h"
+#include "serialization/string.h"
+#include "serialization/containers.h"
 #include "message_transporter.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -112,6 +115,24 @@ namespace mms
     uint32_t round;
     uint32_t signature_count;
     std::string transport_id;
+
+    BEGIN_SERIALIZE_OBJECT()
+      VERSION_FIELD(0)
+      VARINT_FIELD(id)
+      VARINT_FIELD(type)
+      VARINT_FIELD(direction)
+      FIELD(content)
+      VARINT_FIELD(created)
+      VARINT_FIELD(modified)
+      VARINT_FIELD(sent)
+      VARINT_FIELD(signer_index)
+      FIELD(hash)
+      VARINT_FIELD(state)
+      VARINT_FIELD(wallet_height)
+      VARINT_FIELD(round)
+      VARINT_FIELD(signature_count)
+      FIELD(transport_id)
+    END_SERIALIZE()
   };
   // "wallet_height" (for lack of a short name that would describe what it is about)
   // is the number of transfers present in the wallet at the time of message
@@ -132,6 +153,21 @@ namespace mms
     std::string auto_config_transport_address;
     bool auto_config_running;
 
+    BEGIN_SERIALIZE_OBJECT()
+      VERSION_FIELD(0)
+      FIELD(label)
+      FIELD(transport_address)
+      FIELD(monero_address_known)
+      FIELD(monero_address)
+      FIELD(me)
+      VARINT_FIELD(index)
+      FIELD(auto_config_token)
+      FIELD(auto_config_public_key)
+      FIELD(auto_config_secret_key)
+      FIELD(auto_config_transport_address)
+      FIELD(auto_config_running)
+    END_SERIALIZE()
+
     authorized_signer()
     {
       monero_address_known = false;
@@ -164,6 +200,13 @@ namespace mms
     std::string label;
     std::string transport_address;
     cryptonote::account_public_address monero_address;
+
+    BEGIN_SERIALIZE_OBJECT()
+      VERSION_FIELD(0)
+      FIELD(label)
+      FIELD(transport_address)
+      FIELD(monero_address)
+    END_SERIALIZE()
   };
 
   // Overal .mms file structure, with the "message_store" object serialized to and
@@ -174,6 +217,13 @@ namespace mms
     uint32_t file_version;
     crypto::chacha_iv iv;
     std::string encrypted_data;
+
+    BEGIN_SERIALIZE_OBJECT()
+      FIELD(magic_string)
+      FIELD(file_version)
+      FIELD(iv)
+      FIELD(encrypted_data)
+    END_SERIALIZE()
   };
 
     // The following struct provides info about the current state of a "wallet2" object
@@ -198,6 +248,19 @@ namespace mms
     uint32_t multisig_rounds_passed;
     size_t num_transfer_details;
     std::string mms_file;
+
+    BEGIN_SERIALIZE_OBJECT()
+      VERSION_FIELD(0)
+      FIELD(address)
+      VARINT_FIELD(nettype)
+      FIELD(view_secret_key)
+      FIELD(multisig)
+      FIELD(multisig_is_ready)
+      FIELD(has_multisig_partial_key_images)
+      VARINT_FIELD(multisig_rounds_passed)
+      VARINT_FIELD(num_transfer_details)
+      FIELD(mms_file)
+    END_SERIALIZE()
   };
 
   class message_store
@@ -283,7 +346,7 @@ namespace mms
     void stop() { m_run.store(false, std::memory_order_relaxed); m_transporter.stop(); }
 
     void write_to_file(const multisig_wallet_state &state, const std::string &filename);
-    void read_from_file(const multisig_wallet_state &state, const std::string &filename);
+    void read_from_file(const multisig_wallet_state &state, const std::string &filename, bool load_deprecated_formats = false);
 
     template <class t_archive>
     inline void serialize(t_archive &a, const unsigned int ver)
@@ -298,6 +361,18 @@ namespace mms
       a & m_auto_send;
     }
 
+    BEGIN_SERIALIZE_OBJECT()
+      VERSION_FIELD(0)
+      FIELD(m_active)
+      VARINT_FIELD(m_num_authorized_signers)
+      VARINT_FIELD(m_nettype)
+      VARINT_FIELD(m_num_required_signers)
+      FIELD(m_signers)
+      FIELD(m_messages)
+      VARINT_FIELD(m_next_message_id)
+      FIELD(m_auto_send)
+    END_SERIALIZE()
+
     static const char* message_type_to_string(message_type type);
     static const char* message_direction_to_string(message_direction direction);
     static const char* message_state_to_string(message_state state);
diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index 353f20b51..aeddd773d 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -103,8 +103,8 @@ using namespace cryptonote;
 // used to target a given block weight (additional outputs may be added on top to build fee)
 #define TX_WEIGHT_TARGET(bytes) (bytes*2/3)
 
-#define UNSIGNED_TX_PREFIX "Monero unsigned tx set\004"
-#define SIGNED_TX_PREFIX "Monero signed tx set\004"
+#define UNSIGNED_TX_PREFIX "Monero unsigned tx set\005"
+#define SIGNED_TX_PREFIX "Monero signed tx set\005"
 #define MULTISIG_UNSIGNED_TX_PREFIX "Monero multisig unsigned tx set\001"
 
 #define RECENT_OUTPUT_RATIO (0.5) // 50% of outputs are from the recent zone
@@ -1183,6 +1183,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std
   m_offline(false),
   m_rpc_version(0),
   m_export_format(ExportFormat::Binary),
+  m_load_deprecated_formats(false),
   m_credits_target(0)
 {
   set_rpc_client_secret_key(rct::rct2sk(rct::skGen()));
@@ -3903,6 +3904,9 @@ boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee:
   value2.SetInt(m_export_format);
   json.AddMember("export_format", value2, json.GetAllocator());
 
+  value2.SetInt(m_load_deprecated_formats);
+  json.AddMember("load_deprecated_formats", value2, json.GetAllocator());
+
   value2.SetUint(1);
   json.AddMember("encrypted_secret_keys", value2, json.GetAllocator());
 
@@ -4072,6 +4076,7 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st
     m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR;
     m_original_keys_available = false;
     m_export_format = ExportFormat::Binary;
+    m_load_deprecated_formats = false;
     m_device_name = "";
     m_device_derivation_path = "";
     m_key_device_type = hw::device::device_type::SOFTWARE;
@@ -4252,6 +4257,9 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st
     GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, export_format, ExportFormat, Int, false, Binary);
     m_export_format = field_export_format;
 
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, load_deprecated_formats, int, Int, false, false);
+    m_load_deprecated_formats = field_load_deprecated_formats;
+
     GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, device_name, std::string, String, false, std::string());
     if (m_device_name.empty())
     {
@@ -5617,10 +5625,26 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass
       crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cache_data[0]);
 
       try {
-        std::stringstream iss;
-        iss << cache_data;
-        boost::archive::portable_binary_iarchive ar(iss);
-        ar >> *this;
+        bool loaded = false;
+
+        try
+        {
+          std::stringstream iss;
+          iss << cache_data;
+          binary_archive<false> ar(iss);
+          if (::serialization::serialize(ar, *this))
+            if (::serialization::check_stream_state(ar))
+              loaded = true;
+        }
+        catch(...) { }
+
+        if (!loaded)
+        {
+          std::stringstream iss;
+          iss << cache_data;
+          boost::archive::portable_binary_iarchive ar(iss);
+          ar >> *this;
+        }
       }
       catch(...)
       {
@@ -5717,7 +5741,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass
   try
   {
     if (use_fs)
-      m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file);
+      m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file, m_load_deprecated_formats);
   }
   catch (const std::exception &e)
   {
@@ -5907,8 +5931,9 @@ boost::optional<wallet2::cache_file_data> wallet2::get_cache_file_data(const epe
   try
   {
     std::stringstream oss;
-    boost::archive::portable_binary_oarchive ar(oss);
-    ar << *this;
+    binary_archive<true> ar(oss);
+    if (!::serialization::serialize(ar, *this))
+      return boost::none;
 
     boost::optional<wallet2::cache_file_data> cache_file_data = (wallet2::cache_file_data) {};
     cache_file_data.get().cache_data = oss.str();
@@ -6570,10 +6595,11 @@ std::string wallet2::dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) c
   txs.transfers = export_outputs();
   // save as binary
   std::ostringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
+  binary_archive<true> ar(oss);
   try
   {
-    ar << txs;
+    if (!::serialization::serialize(ar, txs))
+      return std::string();
   }
   catch (...)
   {
@@ -6617,6 +6643,11 @@ bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsi
   s = s.substr(1);
   if (version == '\003')
   {
+    if (!m_load_deprecated_formats)
+    {
+      LOG_PRINT_L0("Not loading deprecated format");
+      return false;
+    }
     try
     {
       std::istringstream iss(s);
@@ -6631,6 +6662,11 @@ bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsi
   }
   else if (version == '\004')
   {
+    if (!m_load_deprecated_formats)
+    {
+      LOG_PRINT_L0("Not loading deprecated format");
+      return false;
+    }
     try
     {
       s = decrypt_with_view_secret_key(s);
@@ -6652,6 +6688,26 @@ bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsi
       return false;
     }
   }
+  else if (version == '\005')
+  {
+    try { s = decrypt_with_view_secret_key(s); }
+    catch(const std::exception &e) { LOG_PRINT_L0("Failed to decrypt unsigned tx: " << e.what()); return false; }
+    try
+    {
+      std::istringstream iss(s);
+      binary_archive<false> ar(iss);
+      if (!::serialization::serialize(ar, exported_txs))
+      {
+        LOG_PRINT_L0("Failed to parse data from unsigned tx");
+        return false;
+      }
+    }
+    catch (...)
+    {
+      LOG_PRINT_L0("Failed to parse data from unsigned tx");
+      return false;
+    }
+  }
   else
   {
     LOG_PRINT_L0("Unsupported version in unsigned tx");
@@ -6849,10 +6905,11 @@ std::string wallet2::sign_tx_dump_to_str(unsigned_tx_set &exported_txs, std::vec
 
   // save as binary
   std::ostringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
+  binary_archive<true> ar(oss);
   try
   {
-    ar << signed_txes;
+    if (!::serialization::serialize(ar, signed_txes))
+      return std::string();
   }
   catch(...)
   {
@@ -6901,6 +6958,11 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too
   s = s.substr(1);
   if (version == '\003')
   {
+    if (!m_load_deprecated_formats)
+    {
+      LOG_PRINT_L0("Not loading deprecated format");
+      return false;
+    }
     try
     {
       std::istringstream iss(s);
@@ -6915,6 +6977,11 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too
   }
   else if (version == '\004')
   {
+    if (!m_load_deprecated_formats)
+    {
+      LOG_PRINT_L0("Not loading deprecated format");
+      return false;
+    }
     try
     {
       s = decrypt_with_view_secret_key(s);
@@ -6936,6 +7003,26 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too
       return false;
     }
   }
+  else if (version == '\005')
+  {
+    try { s = decrypt_with_view_secret_key(s); }
+    catch (const std::exception &e) { LOG_PRINT_L0("Failed to decrypt signed transaction: " << e.what()); return false; }
+    try
+    {
+      std::istringstream iss(s);
+      binary_archive<false> ar(iss);
+      if (!::serialization::serialize(ar, signed_txs))
+      {
+        LOG_PRINT_L0("Failed to deserialize signed transaction");
+        return false;
+      }
+    }
+    catch (const std::exception &e)
+    {
+      LOG_PRINT_L0("Failed to decrypt signed transaction: " << e.what());
+      return false;
+    }
+  }
   else
   {
     LOG_PRINT_L0("Unsupported version in signed transaction");
@@ -6987,10 +7074,11 @@ std::string wallet2::save_multisig_tx(multisig_tx_set txs)
 
   // save as binary
   std::ostringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
+  binary_archive<true> ar(oss);
   try
   {
-    ar << txs;
+    if (!::serialization::serialize(ar, txs))
+      return std::string();
   }
   catch (...)
   {
@@ -7054,13 +7142,29 @@ bool wallet2::parse_multisig_tx_from_str(std::string multisig_tx_st, multisig_tx
     LOG_PRINT_L0("Failed to decrypt multisig tx data: " << e.what());
     return false;
   }
+  bool loaded = false;
   try
   {
     std::istringstream iss(multisig_tx_st);
-    boost::archive::portable_binary_iarchive ar(iss);
-    ar >> exported_txs;
+    binary_archive<false> ar(iss);
+    if (::serialization::serialize(ar, exported_txs))
+      if (::serialization::check_stream_state(ar))
+        loaded = true;
   }
-  catch (...)
+  catch (...) {}
+  try
+  {
+    if (!loaded && m_load_deprecated_formats)
+    {
+      std::istringstream iss(multisig_tx_st);
+      boost::archive::portable_binary_iarchive ar(iss);
+      ar >> exported_txs;
+      loaded = true;
+    }
+  }
+  catch(...) {}
+
+  if (!loaded)
   {
     LOG_PRINT_L0("Failed to parse multisig tx data");
     return false;
@@ -9547,8 +9651,8 @@ bool wallet2::light_wallet_parse_rct_str(const std::string& rct_string, const cr
 bool wallet2::light_wallet_key_image_is_ours(const crypto::key_image& key_image, const crypto::public_key& tx_public_key, uint64_t out_index)
 {
   // Lookup key image from cache
-  std::map<uint64_t, crypto::key_image> index_keyimage_map;
-  std::unordered_map<crypto::public_key, std::map<uint64_t, crypto::key_image> >::const_iterator found_pub_key = m_key_image_cache.find(tx_public_key);
+  serializable_map<uint64_t, crypto::key_image> index_keyimage_map;
+  serializable_unordered_map<crypto::public_key, serializable_map<uint64_t, crypto::key_image> >::const_iterator found_pub_key = m_key_image_cache.find(tx_public_key);
   if(found_pub_key != m_key_image_cache.end()) {
     // pub key found. key image for index cached?
     index_keyimage_map = found_pub_key->second;
@@ -11736,7 +11840,7 @@ std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t,
   }
 
   // collect all subaddress spend keys that received those outputs and generate their signatures
-  std::unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys;
+  serializable_unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys;
   for (const cryptonote::subaddress_index &index : subaddr_indices)
   {
     crypto::secret_key subaddr_spend_skey = m_account.get_keys().m_spend_secret_key;
@@ -11753,8 +11857,9 @@ std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t,
 
   // serialize & encode
   std::ostringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
-  ar << proofs << subaddr_spendkeys;
+  binary_archive<true> ar(oss);
+  THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, proofs), error::wallet_internal_error, "Failed to serialize proof");
+  THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, subaddr_spendkeys), error::wallet_internal_error, "Failed to serialize proof");
   return "ReserveProofV2" + tools::base58::encode(oss.str());
 }
 
@@ -11778,11 +11883,25 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr
   THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(std::strlen(header_v1)), sig_decoded), error::wallet_internal_error,
     "Signature decoding error");
 
-  std::istringstream iss(sig_decoded);
-  boost::archive::portable_binary_iarchive ar(iss);
+  bool loaded = false;
   std::vector<reserve_proof_entry> proofs;
-  std::unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys;
-  ar >> proofs >> subaddr_spendkeys;
+  serializable_unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys;
+  try
+  {
+    std::istringstream iss(sig_decoded);
+    binary_archive<false> ar(iss);
+    if (::serialization::serialize_noeof(ar, proofs))
+      if (::serialization::serialize_noeof(ar, subaddr_spendkeys))
+        if (::serialization::check_stream_state(ar))
+          loaded = true;
+  }
+  catch(...) {}
+  if (!loaded && m_load_deprecated_formats)
+  {
+    std::istringstream iss(sig_decoded);
+    boost::archive::portable_binary_iarchive ar(iss);
+    ar >> proofs >> subaddr_spendkeys.parent();
+  }
 
   THROW_WALLET_EXCEPTION_IF(subaddr_spendkeys.count(address.m_spend_public_key) == 0, error::wallet_internal_error,
     "The given address isn't found in the proof");
@@ -12021,7 +12140,7 @@ std::string wallet2::get_description() const
   return "";
 }
 
-const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& wallet2::get_account_tags()
+const std::pair<serializable_map<std::string, std::string>, std::vector<std::string>>& wallet2::get_account_tags()
 {
   // ensure consistency
   if (m_account_tags.second.size() != get_num_subaddress_accounts())
@@ -12743,9 +12862,9 @@ std::string wallet2::export_outputs_to_str(bool all) const
   PERF_TIMER(export_outputs_to_str);
 
   std::stringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
-  const auto& outputs = export_outputs(all);
-  ar << outputs;
+  binary_archive<true> ar(oss);
+  auto outputs = export_outputs(all);
+  THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, outputs), error::wallet_internal_error, "Failed to serialize output data");
 
   std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC));
   const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address;
@@ -12857,23 +12976,39 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st)
   }
 
   size_t imported_outputs = 0;
+  bool loaded = false;
   try
   {
     std::string body(data, headerlen);
-    std::stringstream iss;
-    iss << body;
     std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs;
     try
     {
-      boost::archive::portable_binary_iarchive ar(iss);
-      ar >> outputs;
+      std::stringstream iss;
+      iss << body;
+      binary_archive<false> ar(iss);
+      if (::serialization::serialize(ar, outputs))
+        if (::serialization::check_stream_state(ar))
+          loaded = true;
     }
-    catch (...)
+    catch (...) {}
+
+    if (!loaded && m_load_deprecated_formats)
     {
-      iss.str("");
-      iss << body;
-      boost::archive::binary_iarchive ar(iss);
-      ar >> outputs;
+      try
+      {
+        std::stringstream iss;
+        iss << body;
+        boost::archive::portable_binary_iarchive ar(iss);
+        ar >> outputs;
+        loaded = true;
+      }
+      catch (...) {}
+    }
+
+    if (!loaded)
+    {
+      outputs.first = 0;
+      outputs.second = {};
     }
 
     imported_outputs = import_outputs(outputs);
@@ -13027,8 +13162,8 @@ cryptonote::blobdata wallet2::export_multisig()
   }
 
   std::stringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
-  ar << info;
+  binary_archive<true> ar(oss);
+  CHECK_AND_ASSERT_THROW_MES(::serialization::serialize(ar, info), "Failed to serialize multisig data");
 
   const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address;
   std::string header;
@@ -13098,10 +13233,26 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs)
     seen.insert(signer);
 
     std::string body(data, headerlen);
-    std::istringstream iss(body);
     std::vector<tools::wallet2::multisig_info> i;
-    boost::archive::portable_binary_iarchive ar(iss);
-    ar >> i;
+
+    bool loaded = false;
+    try
+    {
+      std::istringstream iss(body);
+      binary_archive<false> ar(iss);
+      if (::serialization::serialize(ar, i))
+        if (::serialization::check_stream_state(ar))
+          loaded = true;
+    }
+    catch(...) {}
+    if (!loaded && m_load_deprecated_formats)
+    {
+      std::istringstream iss(body);
+      boost::archive::portable_binary_iarchive ar(iss);
+      ar >> i;
+      loaded = true;
+    }
+    CHECK_AND_ASSERT_THROW_MES(loaded, "Failed to load output data");
     MINFO(boost::format("%u outputs found") % boost::lexical_cast<std::string>(i.size()));
     info.push_back(std::move(i));
   }
diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h
index 44ae342c3..1346cd82d 100644
--- a/src/wallet/wallet2.h
+++ b/src/wallet/wallet2.h
@@ -57,7 +57,10 @@
 #include "ringct/rctTypes.h"
 #include "ringct/rctOps.h"
 #include "checkpoints/checkpoints.h"
+#include "serialization/crypto.h"
+#include "serialization/string.h"
 #include "serialization/pair.h"
+#include "serialization/containers.h"
 
 #include "wallet_errors.h"
 #include "common/password.h"
@@ -205,6 +208,13 @@ private:
       a & m_blockchain;
     }
 
+    BEGIN_SERIALIZE_OBJECT()
+      VERSION_FIELD(0)
+      VARINT_FIELD(m_offset)
+      FIELD(m_genesis)
+      FIELD(m_blockchain)
+    END_SERIALIZE()
+
   private:
     size_t m_offset;
     crypto::hash m_genesis;
@@ -381,6 +391,19 @@ private:
       uint64_t m_timestamp;
       bool m_coinbase;
       cryptonote::subaddress_index m_subaddr_index;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(m_tx_hash)
+        VARINT_FIELD(m_amount)
+        FIELD(m_amounts)
+        VARINT_FIELD(m_fee)
+        VARINT_FIELD(m_block_height)
+        VARINT_FIELD(m_unlock_time)
+        VARINT_FIELD(m_timestamp)
+        FIELD(m_coinbase)
+        FIELD(m_subaddr_index)
+      END_SERIALIZE()
     };
 
     struct address_tx : payment_details
@@ -393,6 +416,12 @@ private:
     {
       payment_details m_pd;
       bool m_double_spend_seen;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(m_pd)
+        FIELD(m_double_spend_seen)
+      END_SERIALIZE()
     };
 
     struct unconfirmed_transfer_details
@@ -409,6 +438,21 @@ private:
       uint32_t m_subaddr_account;   // subaddress account of your wallet to be used in this transfer
       std::set<uint32_t> m_subaddr_indices;  // set of address indices used as inputs in this transfer
       std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> m_rings; // relative
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(m_tx)
+        VARINT_FIELD(m_amount_in)
+        VARINT_FIELD(m_amount_out)
+        VARINT_FIELD(m_change)
+        VARINT_FIELD(m_sent_time)
+        FIELD(m_dests)
+        FIELD(m_payment_id)
+        VARINT_FIELD(m_timestamp)
+        VARINT_FIELD(m_subaddr_account)
+        FIELD(m_subaddr_indices)
+        FIELD(m_rings)
+      END_SERIALIZE()
     };
 
     struct confirmed_transfer_details
@@ -428,6 +472,21 @@ private:
       confirmed_transfer_details(): m_amount_in(0), m_amount_out(0), m_change((uint64_t)-1), m_block_height(0), m_payment_id(crypto::null_hash), m_timestamp(0), m_unlock_time(0), m_subaddr_account((uint32_t)-1) {}
       confirmed_transfer_details(const unconfirmed_transfer_details &utd, uint64_t height):
         m_amount_in(utd.m_amount_in), m_amount_out(utd.m_amount_out), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp), m_unlock_time(utd.m_tx.unlock_time), m_subaddr_account(utd.m_subaddr_account), m_subaddr_indices(utd.m_subaddr_indices), m_rings(utd.m_rings) {}
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        VARINT_FIELD(m_amount_in)
+        VARINT_FIELD(m_amount_out)
+        VARINT_FIELD(m_change)
+        VARINT_FIELD(m_block_height)
+        FIELD(m_dests)
+        FIELD(m_payment_id)
+        VARINT_FIELD(m_timestamp)
+        VARINT_FIELD(m_unlock_time)
+        VARINT_FIELD(m_subaddr_account)
+        FIELD(m_subaddr_indices)
+        FIELD(m_rings)
+      END_SERIALIZE()
     };
 
     struct tx_construction_data
@@ -460,7 +519,7 @@ private:
     };
 
     typedef std::vector<transfer_details> transfer_container;
-    typedef std::unordered_multimap<crypto::hash, payment_details> payment_container;
+    typedef serializable_unordered_multimap<crypto::hash, payment_details> payment_container;
 
     struct multisig_sig
     {
@@ -469,6 +528,15 @@ private:
       std::unordered_set<rct::key> used_L;
       std::unordered_set<crypto::public_key> signing_keys;
       rct::multisig_out msout;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(sigs)
+        FIELD(ignore)
+        FIELD(used_L)
+        FIELD(signing_keys)
+        FIELD(msout)
+      END_SERIALIZE()
     };
 
     // The convention for destinations is:
@@ -511,13 +579,26 @@ private:
     {
       std::vector<tx_construction_data> txes;
       std::pair<size_t, wallet2::transfer_container> transfers;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(txes)
+        FIELD(transfers)
+      END_SERIALIZE()
     };
 
     struct signed_tx_set
     {
       std::vector<pending_tx> ptx;
       std::vector<crypto::key_image> key_images;
-      std::unordered_map<crypto::public_key, crypto::key_image> tx_key_images;
+      serializable_unordered_map<crypto::public_key, crypto::key_image> tx_key_images;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(ptx)
+        FIELD(key_images)
+        FIELD(tx_key_images)
+      END_SERIALIZE()
     };
 
     struct multisig_tx_set
@@ -552,7 +633,7 @@ private:
         FIELD(cache_data)
       END_SERIALIZE()
     };
-    
+
     // GUI Address book
     struct address_book_row
     {
@@ -561,6 +642,15 @@ private:
       std::string m_description;   
       bool m_is_subaddress;
       bool m_has_payment_id;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(m_address)
+        FIELD(m_payment_id)
+        FIELD(m_description)
+        FIELD(m_is_subaddress)
+        FIELD(m_has_payment_id)
+      END_SERIALIZE()
     };
 
     struct reserve_proof_entry
@@ -571,6 +661,16 @@ private:
       crypto::key_image key_image;
       crypto::signature shared_secret_sig;
       crypto::signature key_image_sig;
+
+      BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(txid)
+        VARINT_FIELD(index_in_tx)
+        FIELD(shared_secret)
+        FIELD(key_image)
+        FIELD(shared_secret_sig)
+        FIELD(key_image_sig)
+      END_SERIALIZE()
     };
 
     typedef std::tuple<uint64_t, crypto::public_key, rct::key> get_outs_entry;
@@ -927,6 +1027,7 @@ private:
       {
         std::vector<crypto::hash> blockchain;
         a & blockchain;
+        m_blockchain.clear();
         for (const auto &b: blockchain)
         {
           m_blockchain.push_back(b);
@@ -938,25 +1039,25 @@ private:
       }
       a & m_transfers;
       a & m_account_public_address;
-      a & m_key_images;
+      a & m_key_images.parent();
       if(ver < 6)
         return;
-      a & m_unconfirmed_txs;
+      a & m_unconfirmed_txs.parent();
       if(ver < 7)
         return;
-      a & m_payments;
+      a & m_payments.parent();
       if(ver < 8)
         return;
-      a & m_tx_keys;
+      a & m_tx_keys.parent();
       if(ver < 9)
         return;
-      a & m_confirmed_txs;
+      a & m_confirmed_txs.parent();
       if(ver < 11)
         return;
       a & dummy_refresh_height;
       if(ver < 12)
         return;
-      a & m_tx_notes;
+      a & m_tx_notes.parent();
       if(ver < 13)
         return;
       if (ver < 17)
@@ -964,6 +1065,7 @@ private:
         // we're loading an old version, where m_unconfirmed_payments was a std::map
         std::unordered_map<crypto::hash, payment_details> m;
         a & m;
+        m_unconfirmed_payments.clear();
         for (std::unordered_map<crypto::hash, payment_details>::const_iterator i = m.begin(); i != m.end(); ++i)
           m_unconfirmed_payments.insert(std::make_pair(i->first, pool_payment_details{i->second, false}));
       }
@@ -972,6 +1074,7 @@ private:
       if(ver < 15)
       {
         // we're loading an older wallet without a pubkey map, rebuild it
+        m_pub_keys.clear();
         for (size_t i = 0; i < m_transfers.size(); ++i)
         {
           const transfer_details &td = m_transfers[i];
@@ -981,7 +1084,7 @@ private:
         }
         return;
       }
-      a & m_pub_keys;
+      a & m_pub_keys.parent();
       if(ver < 16)
         return;
       a & m_address_book;
@@ -992,6 +1095,7 @@ private:
         // we're loading an old version, where m_unconfirmed_payments payload was payment_details
         std::unordered_multimap<crypto::hash, payment_details> m;
         a & m;
+        m_unconfirmed_payments.clear();
         for (const auto &i: m)
           m_unconfirmed_payments.insert(std::make_pair(i.first, pool_payment_details{i.second, false}));
       }
@@ -1001,20 +1105,20 @@ private:
       a & m_scanned_pool_txs[1];
       if (ver < 20)
         return;
-      a & m_subaddresses;
+      a & m_subaddresses.parent();
       std::unordered_map<cryptonote::subaddress_index, crypto::public_key> dummy_subaddresses_inv;
       a & dummy_subaddresses_inv;
       a & m_subaddress_labels;
-      a & m_additional_tx_keys;
+      a & m_additional_tx_keys.parent();
       if(ver < 21)
         return;
-      a & m_attributes;
+      a & m_attributes.parent();
       if(ver < 22)
         return;
-      a & m_unconfirmed_payments;
+      a & m_unconfirmed_payments.parent();
       if(ver < 23)
         return;
-      a & m_account_tags;
+      a & (std::pair<std::map<std::string, std::string>, std::vector<std::string>>&)m_account_tags;
       if(ver < 24)
         return;
       a & m_ring_history_saved;
@@ -1023,18 +1127,48 @@ private:
       a & m_last_block_reward;
       if(ver < 26)
         return;
-      a & m_tx_device;
+      a & m_tx_device.parent();
       if(ver < 27)
         return;
       a & m_device_last_key_image_sync;
       if(ver < 28)
         return;
-      a & m_cold_key_images;
+      a & m_cold_key_images.parent();
       if(ver < 29)
         return;
       a & m_rpc_client_secret_key;
     }
 
+    BEGIN_SERIALIZE_OBJECT()
+      MAGIC_FIELD("monero wallet cache")
+      VERSION_FIELD(0)
+      FIELD(m_blockchain)
+      FIELD(m_transfers)
+      FIELD(m_account_public_address)
+      FIELD(m_key_images)
+      FIELD(m_unconfirmed_txs)
+      FIELD(m_payments)
+      FIELD(m_tx_keys)
+      FIELD(m_confirmed_txs)
+      FIELD(m_tx_notes)
+      FIELD(m_unconfirmed_payments)
+      FIELD(m_pub_keys)
+      FIELD(m_address_book)
+      FIELD(m_scanned_pool_txs[0])
+      FIELD(m_scanned_pool_txs[1])
+      FIELD(m_subaddresses)
+      FIELD(m_subaddress_labels)
+      FIELD(m_additional_tx_keys)
+      FIELD(m_attributes)
+      FIELD(m_account_tags)
+      FIELD(m_ring_history_saved)
+      FIELD(m_last_block_reward)
+      FIELD(m_tx_device)
+      FIELD(m_device_last_key_image_sync)
+      FIELD(m_cold_key_images)
+      FIELD(m_rpc_client_secret_key)
+    END_SERIALIZE()
+
     /*!
      * \brief  Check if wallet keys and bin files exist
      * \param  file_path           Wallet file path
@@ -1106,6 +1240,8 @@ private:
     void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; }
     const ExportFormat & export_format() const { return m_export_format; }
     inline void set_export_format(const ExportFormat& export_format) { m_export_format = export_format; }
+    bool load_deprecated_formats() const { return m_load_deprecated_formats; }
+    void load_deprecated_formats(bool load) { m_load_deprecated_formats = load; }
     bool persistent_rpc_client_id() const { return m_persistent_rpc_client_id; }
     void persistent_rpc_client_id(bool persistent) { m_persistent_rpc_client_id = persistent; }
     void auto_mine_for_rpc_payment_threshold(float threshold) { m_auto_mine_for_rpc_payment_threshold = threshold; }
@@ -1197,7 +1333,7 @@ private:
      * \brief  Get the list of registered account tags. 
      * \return first.Key=(tag's name), first.Value=(tag's label), second[i]=(i-th account's tag)
      */
-    const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& get_account_tags();
+    const std::pair<serializable_map<std::string, std::string>, std::vector<std::string>>& get_account_tags();
     /*!
      * \brief  Set a tag to the given accounts.
      * \param  account_indices  Indices of accounts.
@@ -1539,28 +1675,28 @@ private:
     std::string m_mms_file;
     const std::unique_ptr<epee::net_utils::http::abstract_http_client> m_http_client;
     hashchain m_blockchain;
-    std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs;
-    std::unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs;
-    std::unordered_multimap<crypto::hash, pool_payment_details> m_unconfirmed_payments;
-    std::unordered_map<crypto::hash, crypto::secret_key> m_tx_keys;
+    serializable_unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs;
+    serializable_unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs;
+    serializable_unordered_multimap<crypto::hash, pool_payment_details> m_unconfirmed_payments;
+    serializable_unordered_map<crypto::hash, crypto::secret_key> m_tx_keys;
     cryptonote::checkpoints m_checkpoints;
-    std::unordered_map<crypto::hash, std::vector<crypto::secret_key>> m_additional_tx_keys;
+    serializable_unordered_map<crypto::hash, std::vector<crypto::secret_key>> m_additional_tx_keys;
 
     transfer_container m_transfers;
     payment_container m_payments;
-    std::unordered_map<crypto::key_image, size_t> m_key_images;
-    std::unordered_map<crypto::public_key, size_t> m_pub_keys;
+    serializable_unordered_map<crypto::key_image, size_t> m_key_images;
+    serializable_unordered_map<crypto::public_key, size_t> m_pub_keys;
     cryptonote::account_public_address m_account_public_address;
-    std::unordered_map<crypto::public_key, cryptonote::subaddress_index> m_subaddresses;
+    serializable_unordered_map<crypto::public_key, cryptonote::subaddress_index> m_subaddresses;
     std::vector<std::vector<std::string>> m_subaddress_labels;
-    std::unordered_map<crypto::hash, std::string> m_tx_notes;
-    std::unordered_map<std::string, std::string> m_attributes;
+    serializable_unordered_map<crypto::hash, std::string> m_tx_notes;
+    serializable_unordered_map<std::string, std::string> m_attributes;
     std::vector<tools::wallet2::address_book_row> m_address_book;
-    std::pair<std::map<std::string, std::string>, std::vector<std::string>> m_account_tags;
+    std::pair<serializable_map<std::string, std::string>, std::vector<std::string>> m_account_tags;
     uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value
     const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info;
     const std::vector<std::vector<rct::key>> *m_multisig_rescan_k;
-    std::unordered_map<crypto::public_key, crypto::key_image> m_cold_key_images;
+    serializable_unordered_map<crypto::public_key, crypto::key_image> m_cold_key_images;
 
     std::atomic<bool> m_run;
 
@@ -1627,7 +1763,7 @@ private:
     uint64_t m_credits_target;
 
     // Aux transaction data from device
-    std::unordered_map<crypto::hash, std::string> m_tx_device;
+    serializable_unordered_map<crypto::hash, std::string> m_tx_device;
 
     // Light wallet
     bool m_light_wallet; /* sends view key to daemon for scanning */
@@ -1641,7 +1777,7 @@ private:
     // We save the info from the first call in m_light_wallet_address_txs for easier lookup.
     std::unordered_map<crypto::hash, address_tx> m_light_wallet_address_txs;
     // store calculated key image for faster lookup
-    std::unordered_map<crypto::public_key, std::map<uint64_t, crypto::key_image> > m_key_image_cache;
+    serializable_unordered_map<crypto::public_key, serializable_map<uint64_t, crypto::key_image> > m_key_image_cache;
 
     std::string m_ring_database;
     bool m_ring_history_saved;
@@ -1668,6 +1804,7 @@ private:
     std::unique_ptr<wallet_device_callback> m_device_callback;
 
     ExportFormat m_export_format;
+    bool m_load_deprecated_formats;
 
     static boost::mutex default_daemon_address_lock;
     static std::string default_daemon_address;
@@ -2064,7 +2201,7 @@ namespace boost
       a & x.key_images;
       if (ver < 1)
         return;
-      a & x.tx_key_images;
+      a & x.tx_key_images.parent();
     }
 
     template <class Archive>
diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp
index 2391b51fd..0ed749cb7 100644
--- a/src/wallet/wallet_rpc_server.cpp
+++ b/src/wallet/wallet_rpc_server.cpp
@@ -838,10 +838,11 @@ namespace tools
   static std::string ptx_to_string(const tools::wallet2::pending_tx &ptx)
   {
     std::ostringstream oss;
-    boost::archive::portable_binary_oarchive ar(oss);
+    binary_archive<true> ar(oss);
     try
     {
-      ar << ptx;
+      if (!::serialization::serialize(ar, const_cast<tools::wallet2::pending_tx&>(ptx)))
+        return "";
     }
     catch (...)
     {
@@ -1550,14 +1551,31 @@ namespace tools
       return false;
     }
 
+    bool loaded = false;
     tools::wallet2::pending_tx ptx;
+
     try
     {
       std::istringstream iss(blob);
-      boost::archive::portable_binary_iarchive ar(iss);
-      ar >> ptx;
+      binary_archive<false> ar(iss);
+      if (::serialization::serialize(ar, ptx))
+        loaded = true;
     }
-    catch (...)
+    catch(...) {}
+
+    if (!loaded && !m_restricted)
+    {
+      try
+      {
+        std::istringstream iss(blob);
+        boost::archive::portable_binary_iarchive ar(iss);
+        ar >> ptx;
+        loaded = true;
+      }
+      catch (...) {}
+    }
+
+    if (!loaded)
     {
       er.code = WALLET_RPC_ERROR_CODE_BAD_TX_METADATA;
       er.message = "Failed to parse tx metadata.";
diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h
index ae861d177..5b2536bf1 100644
--- a/src/wallet/wallet_rpc_server_commands_defs.h
+++ b/src/wallet/wallet_rpc_server_commands_defs.h
@@ -47,7 +47,7 @@
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define WALLET_RPC_VERSION_MAJOR 1
-#define WALLET_RPC_VERSION_MINOR 18
+#define WALLET_RPC_VERSION_MINOR 19
 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR)
 namespace tools
author	moneromooo-monero <moneromooo-monero@users.noreply.github.com>	2020-06-24 23:26:58 +0000
committer	moneromooo-monero <moneromooo-monero@users.noreply.github.com>	2020-08-17 16:23:58 +0000
commit	7175dcb1078abbdaa130a8c5f5fd2b93fa7b3086 (patch)
tree	640b83c4101dc992151630a76a13597913d2dd08 /src
parent	Merge pull request #6736 (diff)
download	monero-7175dcb1078abbdaa130a8c5f5fd2b93fa7b3086.tar.xz