wallet2: fix rescanning tx via scan_tx

- Detach & re-process txs >= lowest scan height
- ensures that if a user calls scan_tx(tx1) after scanning tx2,
the wallet correctly processes tx1 and tx2
- if a user provides a tx with a height higher than the wallet's
last scanned height, the wallet will scan starting from that tx's
height
- scan_tx requires trusted daemon iff need to re-process existing
txs: in addition to querying a daemon for txids, if a user
provides a txid of a tx with height *lower* than any *already*
scanned txs in the wallet, then the wallet will also query the
daemon for all the *higher* txs as well. This is likely
unexpected behavior to a caller, and so to protect a caller from
revealing txid's to an untrusted daemon in an unexpected way,
require the daemon be trusted.

3 files changed, 225 insertions, 8 deletions

diff --git a/tests/README.md b/tests/README.md
index c63294e9b..ea57b258f 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -54,7 +54,7 @@ Functional tests are located under the `tests/functional_tests` directory.
 
 Building all the tests requires installing the following dependencies:
 ```bash
-pip install requests psutil monotonic zmq
+pip install requests psutil monotonic zmq deepdiff
 ```
 
 First, run a regtest daemon in the offline mode and with a fixed difficulty:
diff --git a/tests/functional_tests/CMakeLists.txt b/tests/functional_tests/CMakeLists.txt
index 0daf71b59..306eba073 100644
--- a/tests/functional_tests/CMakeLists.txt
+++ b/tests/functional_tests/CMakeLists.txt
@@ -67,7 +67,7 @@ target_link_libraries(make_test_signature
 monero_add_minimal_executable(cpu_power_test cpu_power_test.cpp)
 find_program(PYTHON3_FOUND python3 REQUIRED)
 
-execute_process(COMMAND ${PYTHON3_FOUND} "-c" "import requests; import psutil; import monotonic; import zmq; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${PYTHON3_FOUND} "-c" "import requests; import psutil; import monotonic; import zmq; import deepdiff; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE)
 if (REQUESTS_OUTPUT STREQUAL "OK")
   add_test(
     NAME    functional_tests_rpc
@@ -76,6 +76,6 @@ if (REQUESTS_OUTPUT STREQUAL "OK")
     NAME    check_missing_rpc_methods
     COMMAND ${PYTHON3_FOUND} "${CMAKE_CURRENT_SOURCE_DIR}/check_missing_rpc_methods.py" "${CMAKE_SOURCE_DIR}")
 else()
-  message(WARNING "functional_tests_rpc and check_missing_rpc_methods skipped, needs the 'requests', 'psutil', 'monotonic', and 'zmq' python modules")
+  message(WARNING "functional_tests_rpc and check_missing_rpc_methods skipped, needs the 'requests', 'psutil', 'monotonic', 'zmq', and 'deepdiff' python modules")
   set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} functional_tests_rpc check_missing_rpc_methods)
 endif()
diff --git a/tests/functional_tests/transfer.py b/tests/functional_tests/transfer.py
index bd80f8f3c..ddc930eb0 100755
--- a/tests/functional_tests/transfer.py
+++ b/tests/functional_tests/transfer.py
@@ -30,6 +30,9 @@
 
 from __future__ import print_function
 import json
+import pprint
+from deepdiff import DeepDiff
+pp = pprint.PrettyPrinter(indent=2)
 
 """Test simple transfers
 """
@@ -37,6 +40,12 @@ import json
 from framework.daemon import Daemon
 from framework.wallet import Wallet
 
+seeds = [
+    'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted',
+    'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout',
+    'dilute gutter certain antics pamphlet macro enjoy left slid guarded bogeys upload nineteen bomb jubilee enhanced irritate turnip eggs swung jukebox loudly reduce sedan slid',
+]
+
 class TransferTest():
     def run_test(self):
         self.reset()
@@ -53,6 +62,7 @@ class TransferTest():
         self.check_rescan()
         self.check_is_key_image_spent()
         self.check_multiple_submissions()
+        self.check_scan_tx()
 
     def reset(self):
         print('Resetting blockchain')
@@ -63,11 +73,6 @@ class TransferTest():
 
     def create(self):
         print('Creating wallets')
-        seeds = [
-          'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted',
-          'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout',
-          'dilute gutter certain antics pamphlet macro enjoy left slid guarded bogeys upload nineteen bomb jubilee enhanced irritate turnip eggs swung jukebox loudly reduce sedan slid',
-        ]
         self.wallet = [None] * len(seeds)
         for i in range(len(seeds)):
             self.wallet[i] = Wallet(idx = i)
@@ -864,5 +869,217 @@ class TransferTest():
         res = self.wallet[0].get_balance()
         assert res.balance == balance
 
+    def check_scan_tx(self):
+        daemon = Daemon()
+
+        print('Testing scan_tx')
+
+        def diff_transfers(actual_transfers, expected_transfers):
+            diff = DeepDiff(actual_transfers, expected_transfers)
+            if diff != {}:
+                pp.pprint(diff)
+            assert diff == {}
+
+        # set up sender_wallet
+        sender_wallet = self.wallet[0]
+        try: sender_wallet.close_wallet()
+        except: pass
+        sender_wallet.restore_deterministic_wallet(seed = seeds[0])
+        sender_wallet.auto_refresh(enable = False)
+        sender_wallet.refresh()
+        res = sender_wallet.get_transfers()
+        out_len = 0 if 'out' not in res else len(res.out)
+        sender_starting_balance = sender_wallet.get_balance().balance
+        amount = 1000000000000
+        assert sender_starting_balance > amount
+
+        # set up receiver_wallet
+        receiver_wallet = self.wallet[1]
+        try: receiver_wallet.close_wallet()
+        except: pass
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1])
+        receiver_wallet.auto_refresh(enable = False)
+        receiver_wallet.refresh()
+        res = receiver_wallet.get_transfers()
+        in_len = 0 if 'in' not in res else len(res['in'])
+        receiver_starting_balance = receiver_wallet.get_balance().balance
+
+        # transfer from sender_wallet to receiver_wallet
+        dst = {'address': '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', 'amount': amount}
+        res = sender_wallet.transfer([dst])
+        assert len(res.tx_hash) == 32*2
+        txid = res.tx_hash
+        assert res.amount == amount
+        assert res.fee > 0
+        fee = res.fee
+
+        expected_sender_balance = sender_starting_balance - (amount + fee)
+        expected_receiver_balance = receiver_starting_balance + amount
+
+        test = 'Checking scan_tx on outgoing pool tx'
+        for attempt in range(2): # test re-scanning
+            print(test + ' (' + ('first attempt' if attempt == 0 else 're-scanning tx') + ')')
+            sender_wallet.scan_tx([txid])
+            res = sender_wallet.get_transfers()
+            assert 'pool' not in res or len(res.pool) == 0
+            if out_len == 0:
+                assert 'out' not in res
+            else:
+                assert len(res.out) == out_len
+            assert len(res.pending) == 1
+            tx = [x for x in res.pending if x.txid == txid]
+            assert len(tx) == 1
+            tx = tx[0]
+            assert tx.amount == amount
+            assert tx.fee == fee
+            assert len(tx.destinations) == 1
+            assert tx.destinations[0].amount == amount
+            assert tx.destinations[0].address == dst['address']
+            assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        test = 'Checking scan_tx on incoming pool tx'
+        for attempt in range(2): # test re-scanning
+            print(test + ' (' + ('first attempt' if attempt == 0 else 're-scanning tx') + ')')
+            receiver_wallet.scan_tx([txid])
+            res = receiver_wallet.get_transfers()
+            assert 'pending' not in res or len(res.pending) == 0
+            if in_len == 0:
+                assert 'in' not in res
+            else:
+                assert len(res['in']) == in_len
+            assert 'pool' in res and len(res.pool) == 1
+            tx = [x for x in res.pool if x.txid == txid]
+            assert len(tx) == 1
+            tx = tx[0]
+            assert tx.amount == amount
+            assert tx.fee == fee
+            assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        # mine the tx
+        height = daemon.generateblocks(dst['address'], 1).height
+        block_header = daemon.getblockheaderbyheight(height = height).block_header
+        miner_txid = block_header.miner_tx_hash
+        expected_receiver_balance += block_header.reward
+
+        print('Checking scan_tx on outgoing tx before refresh')
+        sender_wallet.scan_tx([txid])
+        res = sender_wallet.get_transfers()
+        assert 'pending' not in res or len(res.pending) == 0
+        assert 'pool' not in res or len (res.pool) == 0
+        assert len(res.out) == out_len + 1
+        tx = [x for x in res.out if x.txid == txid]
+        assert len(tx) == 1
+        tx = tx[0]
+        assert tx.amount == amount
+        assert tx.fee == fee
+        assert len(tx.destinations) == 1
+        assert tx.destinations[0].amount == amount
+        assert tx.destinations[0].address == dst['address']
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print('Checking scan_tx on outgoing tx after refresh')
+        sender_wallet.refresh()
+        sender_wallet.scan_tx([txid])
+        diff_transfers(sender_wallet.get_transfers(), res)
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print("Checking scan_tx on outgoing wallet's earliest tx")
+        earliest_height = height
+        earliest_txid = txid
+        for x in res['in']:
+            if x.height < earliest_height:
+                earliest_height = x.height
+                earliest_txid = x.txid
+        sender_wallet.scan_tx([earliest_txid])
+        diff_transfers(sender_wallet.get_transfers(), res)
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        test = 'Checking scan_tx on outgoing wallet restored at current height'
+        for i, out_tx in enumerate(res.out):
+            if 'destinations' in out_tx:
+                del res.out[i]['destinations'] # destinations are not expected after wallet restore
+        out_txids = [x.txid for x in res.out]
+        in_txids = [x.txid for x in res['in']]
+        all_txs = out_txids + in_txids
+        for test_type in ["all txs", "incoming first", "duplicates within", "duplicates across"]:
+            print(test + ' (' + test_type + ')')
+            sender_wallet.close_wallet()
+            sender_wallet.restore_deterministic_wallet(seed = seeds[0], restore_height = height)
+            assert sender_wallet.get_transfers() == {}
+            if test_type == "all txs":
+                sender_wallet.scan_tx(all_txs)
+            elif test_type == "incoming first":
+                sender_wallet.scan_tx(in_txids)
+                sender_wallet.scan_tx(out_txids)
+            # TODO: test_type == "outgoing first"
+            elif test_type == "duplicates within":
+                sender_wallet.scan_tx(all_txs + all_txs)
+            elif test_type == "duplicates across":
+                sender_wallet.scan_tx(all_txs)
+                sender_wallet.scan_tx(all_txs)
+            else:
+                assert True == False
+            diff_transfers(sender_wallet.get_transfers(), res)
+            assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print('Sanity check against outgoing wallet restored at height 0')
+        sender_wallet.close_wallet()
+        sender_wallet.restore_deterministic_wallet(seed = seeds[0], restore_height = 0)
+        sender_wallet.refresh()
+        diff_transfers(sender_wallet.get_transfers(), res)
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print('Checking scan_tx on incoming txs before refresh')
+        receiver_wallet.scan_tx([txid, miner_txid])
+        res = receiver_wallet.get_transfers()
+        assert 'pending' not in res or len(res.pending) == 0
+        assert 'pool' not in res or len (res.pool) == 0
+        assert len(res['in']) == in_len + 2
+        tx = [x for x in res['in'] if x.txid == txid]
+        assert len(tx) == 1
+        tx = tx[0]
+        assert tx.amount == amount
+        assert tx.fee == fee
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print('Checking scan_tx on incoming txs after refresh')
+        receiver_wallet.refresh()
+        receiver_wallet.scan_tx([txid, miner_txid])
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print("Checking scan_tx on incoming wallet's earliest tx")
+        earliest_height = height
+        earliest_txid = txid
+        for x in res['in']:
+            if x.height < earliest_height:
+                earliest_height = x.height
+                earliest_txid = x.txid
+        receiver_wallet.scan_tx([earliest_txid])
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print('Checking scan_tx on incoming wallet restored at current height')
+        txids = [x.txid for x in res['in']]
+        if 'out' in res:
+            txids = txids + [x.txid for x in res.out]
+        receiver_wallet.close_wallet()
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1], restore_height = height)
+        assert receiver_wallet.get_transfers() == {}
+        receiver_wallet.scan_tx(txids)
+        if 'out' in res:
+            for i, out_tx in enumerate(res.out):
+                if 'destinations' in out_tx:
+                    del res.out[i]['destinations'] # destinations are not expected after wallet restore
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print('Sanity check against incoming wallet restored at height 0')
+        receiver_wallet.close_wallet()
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1], restore_height = 0)
+        receiver_wallet.refresh()
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
 if __name__ == '__main__':
     TransferTest().run_test()