Merge pull request #4260

a54dbaee blockchain_blackball: add --force-chain-reaction-pass flag (moneromooo-monero)
44439c32 record blackballs as amount/offset, and add export ability (moneromooo-monero)
4bce935b blockchain_blackball: more optimizations (moneromooo-monero)
b66ba783 blockchain_blackball: do not process duplicate blockchains parts (moneromooo-monero)
639a3c01 blockchain_blackball: make it clear secondary passes are not incremental (moneromooo-monero)
eb8a51be blockchain_blackball: detect spent outputs by partial ring reuse (moneromooo-monero)
d6d276c6 blockchain_blackball: fix chain reaction phase in incremental mode (moneromooo-monero)
2b2a681b blockchain_blackball: avoid false positives for different amounts (moneromooo-monero)
80e4fef3 blockchain_blackball: set transaction looping txn to read only (moneromooo-monero)
4801d6b5 blockchain_blackball: add stats (moneromooo-monero)
846190fd blockchain_blackball: support pre-v2 databases (moneromooo-monero)
daa6cc7d blockchain_blackball: use LMDB for the cache (moneromooo-monero)
50cb370d ringdb: allow blackballing many outputs at once (moneromooo-monero)

4 files changed, 90 insertions, 61 deletions

diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp
index 3f2634c8b..e9fc6866d 100644
--- a/src/wallet/ringdb.cpp
+++ b/src/wallet/ringdb.cpp
@@ -55,6 +55,13 @@ static int compare_hash32(const MDB_val *a, const MDB_val *b)
   return 0;
 }
 
+static int compare_uint64(const MDB_val *a, const MDB_val *b)
+{
+  const uint64_t va = *(const uint64_t*) a->mv_data;
+  const uint64_t vb = *(const uint64_t*) b->mv_data;
+  return va < vb ? -1 : va > vb;
+}
+
 static std::string compress_ring(const std::vector<uint64_t> &ring)
 {
   std::string s;
@@ -146,7 +153,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed)
   MDB_stat mst;
   int ret;
 
-  needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB
+  needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB
 
   ret = mdb_env_info(env, &mei);
   if (ret)
@@ -217,9 +224,9 @@ ringdb::ringdb(std::string filename, const std::string &genesis):
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
   mdb_set_compare(txn, dbi_rings, compare_hash32);
 
-  dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs);
+  dbr = mdb_dbi_open(txn, ("blackballs2-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
-  mdb_set_dupsort(txn, dbi_blackballs, compare_hash32);
+  mdb_set_dupsort(txn, dbi_blackballs, compare_uint64);
 
   dbr = mdb_txn_commit(txn);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
@@ -374,7 +381,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im
   return true;
 }
 
-bool ringdb::blackball_worker(const crypto::public_key &output, int op)
+bool ringdb::blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op)
 {
   MDB_txn *txn;
   MDB_cursor *cursor;
@@ -382,49 +389,61 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
   bool tx_active = false;
   bool ret = true;
 
-  dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack
+  THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output");
+
+  dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
   dbr = mdb_txn_begin(env, NULL, 0, &txn);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
   epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
   tx_active = true;
 
-  MDB_val key = zerokeyval;
-  MDB_val data;
-  data.mv_data = (void*)&output;
-  data.mv_size = sizeof(output);
 
-  switch (op)
+  MDB_val key, data;
+  for (const std::pair<uint64_t, uint64_t> &output: outputs)
+  {
+    key.mv_data = (void*)&output.first;
+    key.mv_size = sizeof(output.first);
+    data.mv_data = (void*)&output.second;
+    data.mv_size = sizeof(output.second);
+
+    switch (op)
+    {
+      case BLACKBALL_BLACKBALL:
+        MDEBUG("Blackballing output " << output.first << "/" << output.second);
+        dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_APPENDDUP);
+        if (dbr == MDB_KEYEXIST)
+          dbr = 0;
+        break;
+      case BLACKBALL_UNBLACKBALL:
+        MDEBUG("Unblackballing output " << output.first << "/" << output.second);
+        dbr = mdb_del(txn, dbi_blackballs, &key, &data);
+        if (dbr == MDB_NOTFOUND)
+          dbr = 0;
+        break;
+      case BLACKBALL_QUERY:
+        dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor);
+        THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr)));
+        dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH);
+        THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr)));
+        ret = dbr != MDB_NOTFOUND;
+        if (dbr == MDB_NOTFOUND)
+          dbr = 0;
+        mdb_cursor_close(cursor);
+        break;
+      case BLACKBALL_CLEAR:
+        break;
+      default:
+        THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op");
+    }
+    THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr)));
+  }
+
+  if (op == BLACKBALL_CLEAR)
   {
-    case BLACKBALL_BLACKBALL:
-      MDEBUG("Blackballing output " << output);
-      dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA);
-      if (dbr == MDB_KEYEXIST)
-        dbr = 0;
-      break;
-    case BLACKBALL_UNBLACKBALL:
-      MDEBUG("Unblackballing output " << output);
-      dbr = mdb_del(txn, dbi_blackballs, &key, &data);
-      if (dbr == MDB_NOTFOUND)
-        dbr = 0;
-      break;
-    case BLACKBALL_QUERY:
-      dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor);
-      THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr)));
-      dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH);
-      THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr)));
-      ret = dbr != MDB_NOTFOUND;
-      if (dbr == MDB_NOTFOUND)
-        dbr = 0;
-      mdb_cursor_close(cursor);
-      break;
-    case BLACKBALL_CLEAR:
-      dbr = mdb_drop(txn, dbi_blackballs, 0);
-      break;
-    default:
-      THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op");
+    dbr = mdb_drop(txn, dbi_blackballs, 0);
+    THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr)));
   }
-  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr)));
 
   dbr = mdb_txn_commit(txn);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr)));
@@ -432,24 +451,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
   return ret;
 }
 
-bool ringdb::blackball(const crypto::public_key &output)
+bool ringdb::blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs)
+{
+  return blackball_worker(outputs, BLACKBALL_BLACKBALL);
+}
+
+bool ringdb::blackball(const std::pair<uint64_t, uint64_t> &output)
 {
-  return blackball_worker(output, BLACKBALL_BLACKBALL);
+  std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output);
+  return blackball_worker(outputs, BLACKBALL_BLACKBALL);
 }
 
-bool ringdb::unblackball(const crypto::public_key &output)
+bool ringdb::unblackball(const std::pair<uint64_t, uint64_t> &output)
 {
-  return blackball_worker(output, BLACKBALL_UNBLACKBALL);
+  std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output);
+  return blackball_worker(outputs, BLACKBALL_UNBLACKBALL);
 }
 
-bool ringdb::blackballed(const crypto::public_key &output)
+bool ringdb::blackballed(const std::pair<uint64_t, uint64_t> &output)
 {
-  return blackball_worker(output, BLACKBALL_QUERY);
+  std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output);
+  return blackball_worker(outputs, BLACKBALL_QUERY);
 }
 
 bool ringdb::clear_blackballs()
 {
-  return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR);
+  return blackball_worker(std::vector<std::pair<uint64_t, uint64_t>>(), BLACKBALL_CLEAR);
 }
 
 }
diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h
index 6b4bce124..7b448b0d7 100644
--- a/src/wallet/ringdb.h
+++ b/src/wallet/ringdb.h
@@ -49,13 +49,14 @@ namespace tools
     bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs);
     bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative);
 
-    bool blackball(const crypto::public_key &output);
-    bool unblackball(const crypto::public_key &output);
-    bool blackballed(const crypto::public_key &output);
+    bool blackball(const std::pair<uint64_t, uint64_t> &output);
+    bool blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs);
+    bool unblackball(const std::pair<uint64_t, uint64_t> &output);
+    bool blackballed(const std::pair<uint64_t, uint64_t> &output);
     bool clear_blackballs();
 
   private:
-    bool blackball_worker(const crypto::public_key &output, int op);
+    bool blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op);
 
   private:
     std::string filename;
diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index 81fe3892c..64a1ec7de 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -6341,7 +6341,7 @@ bool wallet2::find_and_save_rings(bool force)
   return true;
 }
 
-bool wallet2::blackball_output(const crypto::public_key &output)
+bool wallet2::blackball_output(const std::pair<uint64_t, uint64_t> &output)
 {
   if (!m_ringdb)
     return false;
@@ -6349,7 +6349,7 @@ bool wallet2::blackball_output(const crypto::public_key &output)
   catch (const std::exception &e) { return false; }
 }
 
-bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add)
+bool wallet2::set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add)
 {
   if (!m_ringdb)
     return false;
@@ -6358,14 +6358,13 @@ bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &out
     bool ret = true;
     if (!add)
       ret &= m_ringdb->clear_blackballs();
-    for (const auto &output: outputs)
-      ret &= m_ringdb->blackball(output);
+    ret &= m_ringdb->blackball(outputs);
     return ret;
   }
   catch (const std::exception &e) { return false; }
 }
 
-bool wallet2::unblackball_output(const crypto::public_key &output)
+bool wallet2::unblackball_output(const std::pair<uint64_t, uint64_t> &output)
 {
   if (!m_ringdb)
     return false;
@@ -6373,7 +6372,7 @@ bool wallet2::unblackball_output(const crypto::public_key &output)
   catch (const std::exception &e) { return false; }
 }
 
-bool wallet2::is_output_blackballed(const crypto::public_key &output) const
+bool wallet2::is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const
 {
   if (!m_ringdb)
     return false;
@@ -6418,8 +6417,8 @@ bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_out
   CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty");
   if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates
     return false;
-  if (is_output_blackballed(output_public_key)) // don't add blackballed outputs
-    return false;
+//  if (is_output_blackballed(output_public_key)) // don't add blackballed outputs
+//    return false;
   outs.back().push_back(item);
   return true;
 }
@@ -6916,6 +6915,8 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
 
           if (seen_indices.count(i))
             continue;
+          if (is_output_blackballed(std::make_pair(amount, i))) // don't add blackballed outputs
+            continue;
           seen_indices.emplace(i);
 
           LOG_PRINT_L2("picking " << i << " as " << type);
diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h
index f9b516bff..3709e23e0 100644
--- a/src/wallet/wallet2.h
+++ b/src/wallet/wallet2.h
@@ -1159,10 +1159,10 @@ namespace tools
     bool set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative);
     bool find_and_save_rings(bool force = true);
 
-    bool blackball_output(const crypto::public_key &output);
-    bool set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add = false);
-    bool unblackball_output(const crypto::public_key &output);
-    bool is_output_blackballed(const crypto::public_key &output) const;
+    bool blackball_output(const std::pair<uint64_t, uint64_t> &output);
+    bool set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add = false);
+    bool unblackball_output(const std::pair<uint64_t, uint64_t> &output);
+    bool is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const;
 
     bool lock_keys_file();
     bool unlock_keys_file();