storages: overridable limits for loading portable_storage from binary

5 files changed, 63 insertions, 33 deletions

diff --git a/contrib/epee/include/storages/http_abstract_invoke.h b/contrib/epee/include/storages/http_abstract_invoke.h
index c4cb91130..c615b20e6 100644
--- a/contrib/epee/include/storages/http_abstract_invoke.h
+++ b/contrib/epee/include/storages/http_abstract_invoke.h
@@ -98,7 +98,12 @@ namespace epee
         return false;
       }
 
-      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body));
+      static const constexpr epee::serialization::portable_storage::limits_t default_http_bin_limits = {
+        65536 * 3, // objects
+        65536 * 3, // fields
+        65536 * 3, // strings
+      };
+      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body), &default_http_bin_limits);
     }
 
     template<class t_request, class t_response, class t_transport>
diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h
index 95f0bb410..802e16c1b 100644
--- a/contrib/epee/include/storages/levin_abstract_invoke2.h
+++ b/contrib/epee/include/storages/levin_abstract_invoke2.h
@@ -52,6 +52,11 @@ namespace
     snprintf(buf, sizeof(buf),  "command-%u", command);
     return on_levin_traffic(context, initiator, sent, error, bytes, buf);
   }
+  static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = {
+    8192, // objects
+    16384, // fields
+    16384, // strings
+  };
 }
 
 namespace epee
@@ -77,7 +82,7 @@ namespace epee
         return false;
       }
       serialization::portable_storage stg_ret;
-      if(!stg_ret.load_from_binary(buff_to_recv))
+      if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
       {
         LOG_ERROR("Failed to load_from_binary on command " << command);
         return false;
@@ -124,7 +129,7 @@ namespace epee
         return false;
       }
       typename serialization::portable_storage stg_ret;
-      if(!stg_ret.load_from_binary(buff_to_recv))
+      if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
       {
         on_levin_traffic(context, true, false, true, buff_to_recv.size(), command);
         LOG_ERROR("Failed to load_from_binary on command " << command);
@@ -155,7 +160,7 @@ namespace epee
           return false;
         }
         serialization::portable_storage stg_ret;
-        if(!stg_ret.load_from_binary(buff))
+        if(!stg_ret.load_from_binary(buff, &default_levin_limits))
         {
           on_levin_traffic(context, true, false, true, buff.size(), command);
           LOG_ERROR("Failed to load_from_binary on command " << command);
@@ -205,7 +210,7 @@ namespace epee
     int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context )
     {
       serialization::portable_storage strg;
-      if(!strg.load_from_binary(in_buff))
+      if(!strg.load_from_binary(in_buff, &default_levin_limits))
       {
         on_levin_traffic(context, false, false, true, in_buff.size(), command);
         LOG_ERROR("Failed to load_from_binary in command " << command);
@@ -239,7 +244,7 @@ namespace epee
     int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context)
     {
       serialization::portable_storage strg;
-      if(!strg.load_from_binary(in_buff))
+      if(!strg.load_from_binary(in_buff, &default_levin_limits))
       {
         on_levin_traffic(context, false, false, true, in_buff.size(), command);
         LOG_ERROR("Failed to load_from_binary in notify " << command);
diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h
index 2aeadf72c..f77e89cb6 100644
--- a/contrib/epee/include/storages/portable_storage.h
+++ b/contrib/epee/include/storages/portable_storage.h
@@ -54,6 +54,13 @@ namespace epee
       typedef epee::serialization::harray  harray;
       typedef storage_entry meta_entry;
 
+      struct limits_t
+      {
+        size_t n_objects;
+        size_t n_fields;
+        size_t n_strings; // not counting field names
+      };
+
       portable_storage(){}
       virtual ~portable_storage(){}
       hsection   open_section(const std::string& section_name,  hsection hparent_section, bool create_if_notexist = false);
@@ -84,8 +91,8 @@ namespace epee
 
       //-------------------------------------------------------------------------------
       bool		store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192);
-      bool		load_from_binary(const epee::span<const uint8_t> target);
-      bool		load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); }
+      bool		load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL);
+      bool		load_from_binary(const std::string& target, const limits_t *limits = NULL) { return load_from_binary(epee::strspan<uint8_t>(target), limits); }
       template<class trace_policy>
       bool		  dump_as_xml(std::string& targetObj, const std::string& root_name = "");
       bool		  dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true);
@@ -134,7 +141,7 @@ namespace epee
       return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format
     }
     inline
-    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source)
+    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits)
     {
       m_root.m_entries.clear();
       if(source.size() < sizeof(storage_block_header))
@@ -157,6 +164,8 @@ namespace epee
       }
       TRY_ENTRY();
       throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header));
+      if (limits)
+        buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings);
       buf_reader.read(m_root);
       return true;//TODO:
       CATCH_ENTRY("portable_storage::load_from_binary", false);
diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h
index fa3866d87..9e7b6ec34 100644
--- a/contrib/epee/include/storages/portable_storage_from_bin.h
+++ b/contrib/epee/include/storages/portable_storage_from_bin.h
@@ -38,9 +38,6 @@
 #else 
 #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL 100
 #endif
-#define EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL 65536
-#define EPEE_PORTABLE_STORAGE_OBJECT_FIELD_LIMIT_INTERNAL 65536
-#define EPEE_PORTABLE_STORAGE_STRING_LIMIT_INTERNAL 65536 // does not include field names
 
 namespace epee
 {
@@ -49,21 +46,20 @@ namespace epee
     template<typename T>
     struct ps_min_bytes {
       static constexpr const size_t strict = 4096; // actual low bound
-      static constexpr const size_t rough = 4096; // when we want to be stricter for DoS prevention
     };
-    template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8, rough = 8; };
-    template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8, rough = 8; };
-    template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4, rough = 4; };
-    template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4, rough = 4; };
-    template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2, rough = 2; };
-    template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2, rough = 2; };
-    template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1, rough = 1; };
-    template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1, rough = 1; };
-    template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8, rough = 8; };
-    template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1, rough = 1; };
-    template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2, rough = 16; };
-    template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1, rough = 256; };
-    template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1, rough = 128; };
+    template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4; };
+    template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4; };
+    template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1; };
 
     struct throwable_buffer_reader
     {
@@ -86,6 +82,7 @@ namespace epee
       void read(array_entry &ae);
       template<class t_type>
       size_t min_bytes() const;
+      void set_limits(size_t objects, size_t fields, size_t strings);
     private:
       struct recursuion_limitation_guard
       {
@@ -109,6 +106,10 @@ namespace epee
       size_t m_objects;
       size_t m_fields;
       size_t m_strings;
+
+      size_t max_objects;
+      size_t max_fields;
+      size_t max_strings;
     };
 
     inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz)
@@ -123,6 +124,9 @@ namespace epee
       m_objects = 0;
       m_fields = 0;
       m_strings = 0;
+      max_objects = std::numeric_limits<size_t>::max();
+      max_fields = std::numeric_limits<size_t>::max();
+      max_strings = std::numeric_limits<size_t>::max();
     }
     inline 
     void throwable_buffer_reader::read(void* target, size_t count)
@@ -173,12 +177,12 @@ namespace epee
       CHECK_AND_ASSERT_THROW_MES(size <= m_count / ps_min_bytes<type_name>::strict, "Size sanity check failed");
       if (std::is_same<type_name, section>())
       {
-        CHECK_AND_ASSERT_THROW_MES(size <= EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL - m_objects, "Too many objects");
+        CHECK_AND_ASSERT_THROW_MES(size <= max_objects - m_objects, "Too many objects");
         m_objects += size;
       }
       else if (std::is_same<type_name, std::string>())
       {
-        CHECK_AND_ASSERT_THROW_MES(size <= EPEE_PORTABLE_STORAGE_STRING_LIMIT_INTERNAL - m_strings, "Too many strings");
+        CHECK_AND_ASSERT_THROW_MES(size <= max_strings - m_strings, "Too many strings");
         m_strings += size;
       }
 
@@ -247,7 +251,7 @@ namespace epee
     inline storage_entry throwable_buffer_reader::read_se<std::string>()
     {
       RECURSION_LIMITATION();
-      CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= EPEE_PORTABLE_STORAGE_STRING_LIMIT_INTERNAL, "Too many strings");
+      CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= max_strings, "Too many strings");
       m_strings += 1;
       return storage_entry(read<std::string>());
     }
@@ -257,7 +261,7 @@ namespace epee
     inline storage_entry throwable_buffer_reader::read_se<section>()
     {
       RECURSION_LIMITATION();
-      CHECK_AND_ASSERT_THROW_MES(m_objects < EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL, "Too many objects");
+      CHECK_AND_ASSERT_THROW_MES(m_objects < max_objects, "Too many objects");
       ++m_objects;
       section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio
       storage_entry se(std::move(s));
@@ -310,7 +314,7 @@ namespace epee
       RECURSION_LIMITATION();
       sec.m_entries.clear();
       size_t count = read_varint();
-      CHECK_AND_ASSERT_THROW_MES(count <= EPEE_PORTABLE_STORAGE_OBJECT_FIELD_LIMIT_INTERNAL - m_fields, "Too many object fields");
+      CHECK_AND_ASSERT_THROW_MES(count <= max_fields - m_fields, "Too many object fields");
       m_fields += count;
       while(count--)
       {
@@ -340,5 +344,12 @@ namespace epee
       RECURSION_LIMITATION();
       CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported");
     }
+    inline
+    void throwable_buffer_reader::set_limits(size_t objects, size_t fields, size_t strings)
+    {
+      max_objects = objects;
+      max_fields = fields;
+      max_strings = strings;
+    }
   }
 }
diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h
index 39f900c8d..16dd565ec 100644
--- a/contrib/epee/include/storages/portable_storage_template_helper.h
+++ b/contrib/epee/include/storages/portable_storage_template_helper.h
@@ -85,10 +85,10 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff)
+    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff, const epee::serialization::portable_storage::limits_t *limits = NULL)
     {
       portable_storage ps;
-      bool rs = ps.load_from_binary(binary_buff);
+      bool rs = ps.load_from_binary(binary_buff, limits);
       if(!rs)
         return false;