Merge pull request #2434

36c3465b wallet2: do not keep block hashes below last checkpoint (moneromooo-monero)
cdbbe99c checkpoints: add a token checkpoint on testnet (the genesis block) (moneromooo-monero)
493fad80 serialization: add deque serialization (moneromooo-monero)
fa54b205 fix typo in basic and core CMakeLists.txt (moneromooo-monero)
5d65a75b move checkpoints in a separate library (moneromooo-monero)

2 files changed, 496 insertions, 0 deletions

diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp
new file mode 100644
index 000000000..bea392db0
--- /dev/null
+++ b/src/checkpoints/checkpoints.cpp
@@ -0,0 +1,278 @@
+// Copyright (c) 2014-2017, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#include "include_base_utils.h"
+
+using namespace epee;
+
+#include "checkpoints.h"
+
+#include "common/dns_utils.h"
+#include "include_base_utils.h"
+#include "storages/portable_storage_template_helper.h" // epee json include
+#include <sstream>
+#include <random>
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "checkpoints"
+
+namespace cryptonote
+{
+  //---------------------------------------------------------------------------
+  checkpoints::checkpoints()
+  {
+  }
+  //---------------------------------------------------------------------------
+  bool checkpoints::add_checkpoint(uint64_t height, const std::string& hash_str)
+  {
+    crypto::hash h = crypto::null_hash;
+    bool r = epee::string_tools::parse_tpod_from_hex_string(hash_str, h);
+    CHECK_AND_ASSERT_MES(r, false, "Failed to parse checkpoint hash string into binary representation!");
+
+    // return false if adding at a height we already have AND the hash is different
+    if (m_points.count(height))
+    {
+      CHECK_AND_ASSERT_MES(h == m_points[height], false, "Checkpoint at given height already exists, and hash for new checkpoint was different!");
+    }
+    m_points[height] = h;
+    return true;
+  }
+  //---------------------------------------------------------------------------
+  bool checkpoints::is_in_checkpoint_zone(uint64_t height) const
+  {
+    return !m_points.empty() && (height <= (--m_points.end())->first);
+  }
+  //---------------------------------------------------------------------------
+  bool checkpoints::check_block(uint64_t height, const crypto::hash& h, bool& is_a_checkpoint) const
+  {
+    auto it = m_points.find(height);
+    is_a_checkpoint = it != m_points.end();
+    if(!is_a_checkpoint)
+      return true;
+
+    if(it->second == h)
+    {
+      MINFO("CHECKPOINT PASSED FOR HEIGHT " << height << " " << h);
+      return true;
+    }else
+    {
+      MWARNING("CHECKPOINT FAILED FOR HEIGHT " << height << ". EXPECTED HASH: " << it->second << ", FETCHED HASH: " << h);
+      return false;
+    }
+  }
+  //---------------------------------------------------------------------------
+  bool checkpoints::check_block(uint64_t height, const crypto::hash& h) const
+  {
+    bool ignored;
+    return check_block(height, h, ignored);
+  }
+  //---------------------------------------------------------------------------
+  //FIXME: is this the desired behavior?
+  bool checkpoints::is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const
+  {
+    if (0 == block_height)
+      return false;
+
+    auto it = m_points.upper_bound(blockchain_height);
+    // Is blockchain_height before the first checkpoint?
+    if (it == m_points.begin())
+      return true;
+
+    --it;
+    uint64_t checkpoint_height = it->first;
+    return checkpoint_height < block_height;
+  }
+  //---------------------------------------------------------------------------
+  uint64_t checkpoints::get_max_height() const
+  {
+    std::map< uint64_t, crypto::hash >::const_iterator highest = 
+        std::max_element( m_points.begin(), m_points.end(),
+                         ( boost::bind(&std::map< uint64_t, crypto::hash >::value_type::first, _1) < 
+                           boost::bind(&std::map< uint64_t, crypto::hash >::value_type::first, _2 ) ) );
+    return highest->first;
+  }
+  //---------------------------------------------------------------------------
+  const std::map<uint64_t, crypto::hash>& checkpoints::get_points() const
+  {
+    return m_points;
+  }
+
+  bool checkpoints::check_for_conflicts(const checkpoints& other) const
+  {
+    for (auto& pt : other.get_points())
+    {
+      if (m_points.count(pt.first))
+      {
+        CHECK_AND_ASSERT_MES(pt.second == m_points.at(pt.first), false, "Checkpoint at given height already exists, and hash for new checkpoint was different!");
+      }
+    }
+    return true;
+  }
+
+  bool checkpoints::init_default_checkpoints(bool testnet)
+  {
+    if (testnet)
+    {
+      // just use the genesis block on testnet
+      ADD_CHECKPOINT(0,     "48ca7cd3c8de5b6a4d53d2861fbdaedca141553559f9be9520068053cda8430b");
+      return true;
+    }
+    ADD_CHECKPOINT(1,     "771fbcd656ec1464d3a02ead5e18644030007a0fc664c0a964d30922821a8148");
+    ADD_CHECKPOINT(10,    "c0e3b387e47042f72d8ccdca88071ff96bff1ac7cde09ae113dbb7ad3fe92381");
+    ADD_CHECKPOINT(100,   "ac3e11ca545e57c49fca2b4e8c48c03c23be047c43e471e1394528b1f9f80b2d");
+    ADD_CHECKPOINT(1000,  "5acfc45acffd2b2e7345caf42fa02308c5793f15ec33946e969e829f40b03876");
+    ADD_CHECKPOINT(10000, "c758b7c81f928be3295d45e230646de8b852ec96a821eac3fea4daf3fcac0ca2");
+    ADD_CHECKPOINT(22231, "7cb10e29d67e1c069e6e11b17d30b809724255fee2f6868dc14cfc6ed44dfb25");
+    ADD_CHECKPOINT(29556, "53c484a8ed91e4da621bb2fa88106dbde426fe90d7ef07b9c1e5127fb6f3a7f6");
+    ADD_CHECKPOINT(50000, "0fe8758ab06a8b9cb35b7328fd4f757af530a5d37759f9d3e421023231f7b31c");
+    ADD_CHECKPOINT(80000, "a62dcd7b536f22e003ebae8726e9e7276f63d594e264b6f0cd7aab27b66e75e3");
+    ADD_CHECKPOINT(202612, "bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698");
+    ADD_CHECKPOINT(202613, "e2aa337e78df1f98f462b3b1e560c6b914dec47b610698b7b7d1e3e86b6197c2");
+    ADD_CHECKPOINT(202614, "c29e3dc37d8da3e72e506e31a213a58771b24450144305bcba9e70fa4d6ea6fb");
+    ADD_CHECKPOINT(205000, "5d3d7a26e6dc7535e34f03def711daa8c263785f73ec1fadef8a45880fde8063");
+    ADD_CHECKPOINT(220000, "9613f455933c00e3e33ac315cc6b455ee8aa0c567163836858c2d9caff111553");
+    ADD_CHECKPOINT(230300, "bae7a80c46859db355556e3a9204a337ae8f24309926a1312323fdecf1920e61");
+    ADD_CHECKPOINT(230700, "93e631240ceac831da1aebfc5dac8f722c430463024763ebafa888796ceaeedf");
+    ADD_CHECKPOINT(231350, "b5add137199b820e1ea26640e5c3e121fd85faa86a1e39cf7e6cc097bdeb1131");
+    ADD_CHECKPOINT(232150, "955de8e6b6508af2c24f7334f97beeea651d78e9ade3ab18fec3763be3201aa8");
+    ADD_CHECKPOINT(249380, "654fb0a81ce3e5caf7e3264a70f447d4bd07586c08fa50f6638cc54da0a52b2d");
+    ADD_CHECKPOINT(460000, "75037a7aed3e765db96c75bcf908f59d690a5f3390baebb9edeafd336a1c4831");
+    ADD_CHECKPOINT(500000, "2428f0dbe49796be05ed81b347f53e1f7f44aed0abf641446ec2b94cae066b02");
+    ADD_CHECKPOINT(600000, "f5828ebf7d7d1cb61762c4dfe3ccf4ecab2e1aad23e8113668d981713b7a54c5");
+    ADD_CHECKPOINT(700000, "12be9b3d210b93f574d2526abb9c1ab2a881b479131fd0d4f7dac93875f503cd");
+    ADD_CHECKPOINT(825000, "56503f9ad766774b575be3aff73245e9d159be88132c93d1754764f28da2ff60");
+    ADD_CHECKPOINT(900000, "d9958d0e7dcf91a5a7b11de225927bf7efc6eb26240315ce12372be902cc1337");
+    ADD_CHECKPOINT(913193, "5292d5d56f6ba4de33a58d9a34d263e2cb3c6fee0aed2286fd4ac7f36d53c85f");
+    ADD_CHECKPOINT(1000000, "a886ef5149902d8342475fee9bb296341b891ac67c4842f47a833f23c00ed721");
+    ADD_CHECKPOINT(1100000, "3fd720c5c8b3072fc1ccda922dec1ef25f9ed88a1e6ad4103d0fe00b180a5903");
+    ADD_CHECKPOINT(1150000, "1dd16f626d18e1e988490dfd06de5920e22629c972c58b4d8daddea0038627b2");
+    ADD_CHECKPOINT(1200000, "fa7d13a90850882060479d100141ff84286599ae39c3277c8ea784393f882d1f");
+    ADD_CHECKPOINT(1300000, "31b34272343a44a9f4ac7de7a8fcf3b7d8a3124d7d6870affd510d2f37e74cd0");
+    ADD_CHECKPOINT(1390000, "a8f5649dd4ded60eedab475f2bec8c934681c07e3cf640e9be0617554f13ff6c");
+    
+
+    return true;
+  }
+
+  bool checkpoints::load_checkpoints_from_json(const std::string json_hashfile_fullpath)
+  {
+    boost::system::error_code errcode;
+    if (! (boost::filesystem::exists(json_hashfile_fullpath, errcode)))
+    {
+      LOG_PRINT_L1("Blockchain checkpoints file not found");
+      return true;
+    }
+
+    LOG_PRINT_L1("Adding checkpoints from blockchain hashfile");
+
+    uint64_t prev_max_height = get_max_height();
+    LOG_PRINT_L1("Hard-coded max checkpoint height is " << prev_max_height);
+    t_hash_json hashes;
+    epee::serialization::load_t_from_json_file(hashes, json_hashfile_fullpath);
+    for (std::vector<t_hashline>::const_iterator it = hashes.hashlines.begin(); it != hashes.hashlines.end(); )
+    {
+      uint64_t height;
+      height = it->height;
+      if (height <= prev_max_height) {
+	LOG_PRINT_L1("ignoring checkpoint height " << height);
+      } else {
+	std::string blockhash = it->hash;
+	LOG_PRINT_L1("Adding checkpoint height " << height << ", hash=" << blockhash);
+	ADD_CHECKPOINT(height, blockhash);
+      }
+      ++it;
+    }
+
+    return true;
+  }
+
+  bool checkpoints::load_checkpoints_from_dns(bool testnet)
+  {
+    std::vector<std::string> records;
+
+    // All four MoneroPulse domains have DNSSEC on and valid
+    static const std::vector<std::string> dns_urls = { "checkpoints.moneropulse.se"
+						     , "checkpoints.moneropulse.org"
+						     , "checkpoints.moneropulse.net"
+						     , "checkpoints.moneropulse.co"
+    };
+
+    static const std::vector<std::string> testnet_dns_urls = { "testpoints.moneropulse.se"
+							     , "testpoints.moneropulse.org"
+							     , "testpoints.moneropulse.net"
+							     , "testpoints.moneropulse.co"
+    };
+
+    if (!tools::dns_utils::load_txt_records_from_dns(records, testnet ? testnet_dns_urls : dns_urls))
+      return true; // why true ?
+
+    for (const auto& record : records)
+    {
+      auto pos = record.find(":");
+      if (pos != std::string::npos)
+      {
+        uint64_t height;
+        crypto::hash hash;
+
+        // parse the first part as uint64_t,
+        // if this fails move on to the next record
+        std::stringstream ss(record.substr(0, pos));
+        if (!(ss >> height))
+        {
+    continue;
+        }
+
+        // parse the second part as crypto::hash,
+        // if this fails move on to the next record
+        std::string hashStr = record.substr(pos + 1);
+        if (!epee::string_tools::parse_tpod_from_hex_string(hashStr, hash))
+        {
+    continue;
+        }
+
+        ADD_CHECKPOINT(height, hashStr);
+      }
+    }
+    return true;
+  }
+
+  bool checkpoints::load_new_checkpoints(const std::string json_hashfile_fullpath, bool testnet, bool dns)
+  {
+    bool result;
+
+    result = load_checkpoints_from_json(json_hashfile_fullpath);
+    if (dns)
+    {
+      result &= load_checkpoints_from_dns(testnet);
+    }
+
+    return result;
+  }
+}
diff --git a/src/checkpoints/checkpoints.h b/src/checkpoints/checkpoints.h
new file mode 100644
index 000000000..a643c5790
--- /dev/null
+++ b/src/checkpoints/checkpoints.h
@@ -0,0 +1,218 @@
+// Copyright (c) 2014-2017, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once
+#include <map>
+#include <vector>
+#include "misc_log_ex.h"
+#include "crypto/hash.h"
+#include "serialization/keyvalue_serialization.h"
+
+#define ADD_CHECKPOINT(h, hash)  CHECK_AND_ASSERT(add_checkpoint(h,  hash), false);
+#define JSON_HASH_FILE_NAME "checkpoints.json"
+
+
+namespace cryptonote
+{
+  /**
+   * @brief A container for blockchain checkpoints
+   *
+   * A checkpoint is a pre-defined hash for the block at a given height.
+   * Some of these are compiled-in, while others can be loaded at runtime
+   * either from a json file or via DNS from a checkpoint-hosting server.
+   */
+  class checkpoints
+  {
+  public:
+
+    /**
+     * @brief default constructor
+     */
+    checkpoints();
+
+    /**
+     * @brief adds a checkpoint to the container
+     *
+     * @param height the height of the block the checkpoint is for
+     * @param hash_str the hash of the block, as a string
+     *
+     * @return false if parsing the hash fails, or if the height is a duplicate
+     *         AND the existing checkpoint hash does not match the new one,
+     *         otherwise returns true
+     */
+    bool add_checkpoint(uint64_t height, const std::string& hash_str);
+
+    /**
+     * @brief checks if there is a checkpoint in the future
+     *
+     * This function checks if the height passed is lower than the highest
+     * checkpoint.
+     *
+     * @param height the height to check against
+     *
+     * @return false if no checkpoints, otherwise returns whether or not
+     *         the height passed is lower than the highest checkpoint.
+     */
+    bool is_in_checkpoint_zone(uint64_t height) const;
+
+    /**
+     * @brief checks if the given height and hash agree with the checkpoints
+     *
+     * This function checks if the given height and hash exist in the
+     * checkpoints container.  If so, it returns whether or not the passed
+     * parameters match the stored values.
+     *
+     * @param height the height to be checked
+     * @param h the hash to be checked
+     * @param is_a_checkpoint return-by-reference if there is a checkpoint at the given height
+     *
+     * @return true if there is no checkpoint at the given height,
+     *         true if the passed parameters match the stored checkpoint,
+     *         false otherwise
+     */
+    bool check_block(uint64_t height, const crypto::hash& h, bool& is_a_checkpoint) const;
+
+    /**
+     * @overload
+     */
+    bool check_block(uint64_t height, const crypto::hash& h) const;
+
+    /**
+     * @brief checks if alternate chain blocks should be kept for a given height
+     *
+     * this basically says if the blockchain is smaller than the first
+     * checkpoint then alternate blocks are allowed.  Alternatively, if the
+     * last checkpoint *before* the end of the current chain is also before
+     * the block to be added, then this is fine.
+     *
+     * @param blockchain_height the current blockchain height
+     * @param block_height the height of the block to be added as alternate
+     *
+     * @return true if alternate blocks are allowed given the parameters,
+     *         otherwise false
+     */
+    bool is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const;
+
+    /**
+     * @brief gets the highest checkpoint height
+     *
+     * @return the height of the highest checkpoint
+     */
+    uint64_t get_max_height() const;
+
+    /**
+     * @brief gets the checkpoints container
+     *
+     * @return a const reference to the checkpoints container
+     */
+    const std::map<uint64_t, crypto::hash>& get_points() const;
+
+    /**
+     * @brief checks if our checkpoints container conflicts with another
+     *
+     * A conflict refers to a case where both checkpoint sets have a checkpoint
+     * for a specific height but their hashes for that height do not match.
+     *
+     * @param other the other checkpoints instance to check against
+     *
+     * @return false if any conflict is found, otherwise true
+     */
+    bool check_for_conflicts(const checkpoints& other) const;
+
+    /**
+     * @brief loads the default main chain checkpoints
+     * @param testnet whether to load testnet checkpoints or mainnet
+     *
+     * @return true unless adding a checkpoint fails
+     */
+    bool init_default_checkpoints(bool testnet);
+
+    /**
+     * @brief load new checkpoints
+     *
+     * Loads new checkpoints from the specified json file, as well as
+     * (optionally) from DNS.
+     *
+     * @param json_hashfile_fullpath path to the json checkpoints file
+     * @param testnet whether to load testnet checkpoints or mainnet
+     * @param dns whether or not to load DNS checkpoints
+     *
+     * @return true if loading successful and no conflicts
+     */
+    bool load_new_checkpoints(const std::string json_hashfile_fullpath, bool testnet=false, bool dns=true);
+
+    /**
+     * @brief load new checkpoints from json
+     *
+     * @param json_hashfile_fullpath path to the json checkpoints file
+     *
+     * @return true if loading successful and no conflicts
+     */
+    bool load_checkpoints_from_json(const std::string json_hashfile_fullpath);
+
+    /**
+     * @brief load new checkpoints from DNS
+     *
+     * @param testnet whether to load testnet checkpoints or mainnet
+     *
+     * @return true if loading successful and no conflicts
+     */
+    bool load_checkpoints_from_dns(bool testnet = false);
+
+  private:
+
+
+    /**
+     * @brief struct for loading a checkpoint from json
+     */
+  struct t_hashline
+  {
+    uint64_t height; //!< the height of the checkpoint
+    std::string hash; //!< the hash for the checkpoint
+        BEGIN_KV_SERIALIZE_MAP()
+          KV_SERIALIZE(height)
+          KV_SERIALIZE(hash)
+        END_KV_SERIALIZE_MAP()
+  };
+
+  /**
+   * @brief struct for loading many checkpoints from json
+   */
+  struct t_hash_json {
+    std::vector<t_hashline> hashlines; //!< the checkpoint lines from the file
+        BEGIN_KV_SERIALIZE_MAP()
+          KV_SERIALIZE(hashlines)
+        END_KV_SERIALIZE_MAP()
+  };
+
+    std::map<uint64_t, crypto::hash> m_points; //!< the checkpoints container
+
+  };
+}