diff options
Diffstat (limited to 'src')
45 files changed, 3556 insertions, 1236 deletions
diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index a7fa556bd..b11570e37 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -135,7 +135,7 @@ const unsigned int DB_BUFFER_LENGTH = 32 * MB; const unsigned int DB_DEF_CACHESIZE = 256 * MB; #if defined(BDB_BULK_CAN_THREAD) -const unsigned int DB_BUFFER_COUNT = boost::thread::hardware_concurrency(); +const unsigned int DB_BUFFER_COUNT = tools::get_max_concurrency(); #else const unsigned int DB_BUFFER_COUNT = 1; #endif diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index a66f4a403..68f635d18 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -82,14 +82,17 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } } - add_transaction_data(blk_hash, tx, tx_hash); + uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash); + + std::vector<uint64_t> amount_output_indices; // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index for (uint64_t i = 0; i < tx.vout.size(); ++i) { - add_output(tx_hash, tx.vout[i], i, tx.unlock_time); + amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time)); } + add_tx_amount_output_indices(tx_id, amount_output_indices); } uint64_t BlockchainDB::add_block( const block& blk diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 3585bd061..1445dd13c 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -59,6 +59,39 @@ * Unspent transaction outputs are duplicated to quickly gather random * outputs to use for mixins * + * Indices and Identifiers: + * The word "index" is used ambiguously throughout this code. It is + * particularly confusing when talking about the output or transaction + * tables since their indexing can refer to themselves or each other. + * I have attempted to clarify these usages here: + * + * Blocks, transactions, and outputs are all identified by a hash. + * For storage efficiency, a 64-bit integer ID is used instead of the hash + * inside the DB. Tables exist to map between hash and ID. A block ID is + * also referred to as its "height". Transactions and outputs generally are + * not referred to by ID outside of this module, but the tx ID is returned + * by tx_exists() and used by get_tx_amount_output_indices(). Like their + * corresponding hashes, IDs are globally unique. + * + * The remaining uses of the word "index" refer to local offsets, and are + * not globally unique. An "amount output index" N refers to the Nth output + * of a specific amount. An "output local index" N refers to the Nth output + * of a specific tx. + * + * Exceptions: + * DB_ERROR -- generic + * DB_OPEN_FAILURE + * DB_CREATE_FAILURE + * DB_SYNC_FAILURE + * BLOCK_DNE + * BLOCK_PARENT_DNE + * BLOCK_EXISTS + * BLOCK_INVALID -- considering making this multiple errors + * TX_DNE + * TX_EXISTS + * OUTPUT_DNE + * OUTPUT_EXISTS + * KEY_IMAGE_EXISTS */ namespace cryptonote @@ -80,6 +113,15 @@ struct output_data_t }; #pragma pack(pop) +#pragma pack(push, 1) +struct tx_data_t +{ + uint64_t tx_id; + uint64_t unlock_time; + uint64_t block_id; +}; +#pragma pack(pop) + /*********************************** * Exception Definitions ***********************************/ @@ -311,14 +353,18 @@ private: * and the other data passed here, not the separate outputs of the * transaction. * + * It returns a tx ID, which is a mapping from the tx_hash. The tx ID + * is used in #add_tx_amount_output_indices(). + * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION * * @param blk_hash the hash of the block containing the transaction * @param tx the transaction to be added * @param tx_hash the hash of the transaction + * @return the transaction ID */ - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; /** * @brief remove data about a transaction @@ -348,6 +394,9 @@ private: * future, this tracking (of the number, at least) should be moved to * this class, as it is necessary and the same among all BlockchainDB. * + * It returns an amount output index, which is the index of the output + * for its specified amount. + * * This data should be stored in such a manner that the only thing needed to * reverse the process is the tx_out. * @@ -358,25 +407,24 @@ private: * @param tx_output the output * @param local_index index of the output in its transaction * @param unlock_time unlock time/height of the output + * @return amount output index */ - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0; + virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0; /** - * @brief remove an output - * - * The subclass implementing this will remove all output data it stored - * in add_output(). + * @brief store amount output indices for a tx's outputs * - * In addition, the subclass is responsible for correctly decrementing - * its global output counter (this may be automatic for some, such as using - * a database backend "count" feature). + * The subclass implementing this will add the amount output indices to its + * backing store in a suitable manner. The tx_id will be the same one that + * was returned from #add_output(). * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION * - * @param tx_output the output to be removed + * @param tx_id ID of the transaction containing these outputs + * @param amount_output_indices the amount output indices of the transaction */ - virtual void remove_output(const tx_out& tx_output) = 0; + virtual void add_tx_amount_output_indices(const uint64_t tx_id, const std::vector<uint64_t>& amount_output_indices) = 0; /** * @brief store a spent key @@ -414,18 +462,6 @@ private: */ void pop_block(); - /** - * @brief helper function for add_transactions, to add each individual transaction - * - * This function is called by add_transactions() for each transaction to be - * added. - * - * @param blk_hash hash of the block which has the transaction - * @param tx the transaction to add - * @param tx_hash_ptr the hash of the transaction, if already calculated - */ - void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); - // helper function to remove transaction from blockchain /** * @brief helper function to remove transaction from the blockchain @@ -444,6 +480,18 @@ private: protected: + /** + * @brief helper function for add_transactions, to add each individual transaction + * + * This function is called by add_transactions() for each transaction to be + * added. + * + * @param blk_hash hash of the block which has the transaction + * @param tx the transaction to add + * @param tx_hash_ptr the hash of the transaction, if already calculated + */ + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); + mutable uint64_t time_tx_exists = 0; //!< a performance metric uint64_t time_commit1 = 0; //!< a performance metric bool m_auto_remove_logs = true; //!< whether or not to automatically remove old logs @@ -930,10 +978,12 @@ public: * given hash and return true if so, false otherwise. * * @param h the hash to check against + * @param tx_id (optional) returns the tx_id for the tx hash * * @return true if the transaction exists, otherwise false */ virtual bool tx_exists(const crypto::hash& h) const = 0; + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_id) const = 0; // return unlock time of tx with hash <h> /** @@ -1125,36 +1175,20 @@ public: virtual bool can_thread_bulk_indices() const = 0; /** - * @brief gets output indices (global) for a transaction's outputs - * - * The subclass should fetch the global output indices for each output - * in the transaction with the given hash. - * - * If the transaction does not exist, the subclass should throw TX_DNE. - * - * If an output cannot be found, the subclass should throw OUTPUT_DNE. - * - * @param h a transaction hash - * - * @return a list of global output indices - */ - virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const = 0; - - /** * @brief gets output indices (amount-specific) for a transaction's outputs * * The subclass should fetch the amount-specific output indices for each - * output in the transaction with the given hash. + * output in the transaction with the given ID. * * If the transaction does not exist, the subclass should throw TX_DNE. * * If an output cannot be found, the subclass should throw OUTPUT_DNE. * - * @param h a transaction hash + * @param tx_id a transaction ID * * @return a list of amount-specific output indices */ - virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const = 0; + virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0; /** * @brief check if a key image is stored as spent diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 9b99520a1..8c51c09b1 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -46,7 +46,7 @@ using epee::string_tools::pod_to_hex; // Increase when the DB changes in a non backward compatible way, and there // is no automatic conversion, so that a full resync is needed. -#define VERSION 0 +#define VERSION 1 namespace { @@ -65,6 +65,8 @@ inline void throw1(const T &e) throw e; } +#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} + template<typename T> struct MDB_val_copy: public MDB_val { @@ -110,18 +112,10 @@ private: int compare_uint64(const MDB_val *a, const MDB_val *b) { -#ifdef MISALIGNED_OK - const uint64_t va = *(const uint64_t*)a->mv_data; - const uint64_t vb = *(const uint64_t*)b->mv_data; -#else - uint64_t va, vb; - memcpy(&va, a->mv_data, sizeof(uint64_t)); - memcpy(&vb, b->mv_data, sizeof(uint64_t)); -#endif - if (va < vb) return -1; - else if (va == vb) return 0; - else return 1; -}; + const uint64_t va = *(const uint64_t *)a->mv_data; + const uint64_t vb = *(const uint64_t *)b->mv_data; + return (va < vb) ? -1 : va > vb; +} int compare_uint8(const MDB_val *a, const MDB_val *b) { @@ -151,23 +145,41 @@ int compare_string(const MDB_val *a, const MDB_val *b) return strcmp(va, vb); } +/* DB schema: + * + * Table Key Data + * ----- --- ---- + * blocks block ID block blob + * block_heights block hash block height + * block_info block ID {block metadata} + * + * txs txn ID txn blob + * tx_indices txn hash {txn ID, metadata} + * tx_outputs txn ID [txn amount output indices] + * + * output_txs output ID {txn hash, local index} + * output_amounts amount [{amount output index, metadata}...] + * + * spent_keys input hash - + * + * Note: where the data items are of uniform size, DUPFIXED tables have + * been used to save space. In most of these cases, a dummy "zerokval" + * key is used when accessing the table; the Key listed above will be + * attached as a prefix on the Data to serve as the DUPSORT key. + * (DUPFIXED saves 8 bytes per record.) + * + * The output_amounts table doesn't use a dummy key, but uses DUPSORT. + */ const char* const LMDB_BLOCKS = "blocks"; -const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; -const char* const LMDB_BLOCK_HASHES = "block_hashes"; -const char* const LMDB_BLOCK_SIZES = "block_sizes"; -const char* const LMDB_BLOCK_DIFFS = "block_diffs"; -const char* const LMDB_BLOCK_COINS = "block_coins"; +const char* const LMDB_BLOCK_INFO = "block_info"; const char* const LMDB_TXS = "txs"; -const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; -const char* const LMDB_TX_HEIGHTS = "tx_heights"; +const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; const char* const LMDB_OUTPUT_TXS = "output_txs"; -const char* const LMDB_OUTPUT_INDICES = "output_indices"; const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts"; -const char* const LMDB_OUTPUT_KEYS = "output_keys"; const char* const LMDB_SPENT_KEYS = "spent_keys"; const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights"; @@ -175,6 +187,8 @@ const char* const LMDB_HF_VERSIONS = "hf_versions"; const char* const LMDB_PROPERTIES = "properties"; +const char zerokey[8] = {0}; +const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; const std::string lmdb_error(const std::string& error_string, int mdb_res) { @@ -214,6 +228,39 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { + +typedef struct mdb_block_info +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_size; // a size_t really but we need 32-bit compat + difficulty_type bi_diff; + crypto::hash bi_hash; +} mdb_block_info; + +typedef struct blk_height { + crypto::hash bh_hash; + uint64_t bh_height; +} blk_height; + +typedef struct txindex { + crypto::hash key; + tx_data_t data; +} txindex; + +typedef struct outkey { + uint64_t amount_index; + uint64_t output_id; + output_data_t data; +} outkey; + +typedef struct outtx { + uint64_t output_id; + crypto::hash tx_hash; + uint64_t local_index; +} outtx; + std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0}; std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; @@ -377,7 +424,9 @@ void BlockchainLMDB::do_resize(uint64_t increase_size) mdb_txn_safe::wait_no_active_txns(); - mdb_env_set_mapsize(m_env, new_mapsize); + int result = mdb_env_set_mapsize(m_env, new_mapsize); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to set new mapsize: ", result).c_str())); LOG_PRINT_GREEN("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB", LOG_LEVEL_0); @@ -537,70 +586,55 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(block_heights) - MDB_val_copy<crypto::hash> val_h(blk_hash); - if (mdb_cursor_get(m_cur_block_heights, &val_h, NULL, MDB_SET) == 0) + blk_height bh = {blk_hash, m_height}; + MDB_val_set(val_h, bh); + if (mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); if (m_height > 0) { - MDB_val_copy<crypto::hash> parent_key(blk.prev_id); - MDB_val parent_h; - int result = mdb_cursor_get(m_cur_block_heights, &parent_key, &parent_h, MDB_SET); + MDB_val_set(parent_key, blk.prev_id); + int result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &parent_key, MDB_GET_BOTH); if (result) { LOG_PRINT_L3("m_height: " << m_height); LOG_PRINT_L3("parent_key: " << blk.prev_id); throw0(DB_ERROR(lmdb_error("Failed to get top block hash to check for new block's parent: ", result).c_str())); } - uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; - if (parent_height != m_height - 1) + blk_height *prev = (blk_height *)parent_key.mv_data; + if (prev->bh_height != m_height - 1) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); } int result = 0; - MDB_val_copy<uint64_t> key(m_height); + MDB_val_set(key, m_height); CURSOR(blocks) - CURSOR(block_sizes) - CURSOR(block_timestamps) - CURSOR(block_diffs) - CURSOR(block_coins) - CURSOR(block_hashes) + CURSOR(block_info) MDB_val_copy<blobdata> blob(block_to_blob(blk)); result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str())); - MDB_val_copy<size_t> sz(block_size); - result = mdb_cursor_put(m_cur_block_sizes, &key, &sz, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block size to db transaction: ", result).c_str())); + mdb_block_info bi; + bi.bi_height = m_height; + bi.bi_timestamp = blk.timestamp; + bi.bi_coins = coins_generated; + bi.bi_size = block_size; + bi.bi_diff = cumulative_difficulty; + bi.bi_hash = blk_hash; - MDB_val_copy<uint64_t> ts(blk.timestamp); - result = mdb_cursor_put(m_cur_block_timestamps, &key, &ts, MDB_APPEND); + MDB_val_set(val, bi); + result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block timestamp to db transaction: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block info to db transaction: ", result).c_str())); - MDB_val_copy<difficulty_type> diff(cumulative_difficulty); - result = mdb_cursor_put(m_cur_block_diffs, &key, &diff, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block cumulative difficulty to db transaction: ", result).c_str())); - - MDB_val_copy<uint64_t> coinsgen(coins_generated); - result = mdb_cursor_put(m_cur_block_coins, &key, &coinsgen, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block total generated coins to db transaction: ", result).c_str())); - - result = mdb_cursor_put(m_cur_block_heights, &val_h, &key, 0); + result = mdb_cursor_put(m_cur_block_heights, (MDB_val *)&zerokval, &val_h, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); - result = mdb_cursor_put(m_cur_block_hashes, &key, &val_h, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add block hash to db transaction: ", result).c_str())); - m_cum_size += block_size; m_cum_count++; } @@ -615,66 +649,80 @@ void BlockchainLMDB::remove_block() if (m_height == 0) throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(block_info) + CURSOR(block_heights) + CURSOR(blocks) MDB_val_copy<uint64_t> k(m_height - 1); - MDB_val h; - if ((result = mdb_get(*m_write_txn, m_block_hashes, &k, &h))) + MDB_val h = k; + if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) throw1(BLOCK_DNE(lmdb_error("Attempting to remove block that's not in the db: ", result).c_str())); - if ((result = mdb_del(*m_write_txn, m_blocks, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_sizes, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block size to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_diffs, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block cumulative difficulty to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_coins, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block total generated coins to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_timestamps, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block timestamp to db transaction: ", result).c_str())); - - if ((result = mdb_del(*m_write_txn, m_block_heights, &h, NULL))) + // must use h now; deleting from m_block_info will invalidate it + mdb_block_info *bi = (mdb_block_info *)h.mv_data; + blk_height bh = {bi->bi_hash, 0}; + h.mv_data = (void *)&bh; + h.mv_size = sizeof(bh); + if ((result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) + throw1(DB_ERROR(lmdb_error("Failed to locate block height by hash for removal: ", result).c_str())); + if ((result = mdb_cursor_del(m_cur_block_heights, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str())); - if ((result = mdb_del(*m_write_txn, m_block_hashes, &k, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of block hash to db transaction: ", result).c_str())); + if ((result = mdb_cursor_get(m_cur_blocks, &k, NULL, MDB_SET))) + throw1(DB_ERROR(lmdb_error("Failed to locate block for removal: ", result).c_str())); + if ((result = mdb_cursor_del(m_cur_blocks, 0))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); + + if ((result = mdb_cursor_del(m_cur_block_info, 0))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str())); } -void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); mdb_txn_cursors *m_cursors = &m_wcursors; - int result = 0; + int result; + uint64_t tx_id = m_num_txs; CURSOR(txs) - CURSOR(tx_heights) - CURSOR(tx_unlocks) + CURSOR(tx_indices) + + MDB_val_set(val_tx_id, tx_id); + MDB_val_set(val_h, tx_hash); + result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); + if (result == 0) { + txindex *tip = (txindex *)val_h.mv_data; + throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx id ").append(boost::lexical_cast<std::string>(tip->data.tx_id)).append(")").c_str())); + } else if (result != MDB_NOTFOUND) { + throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); + } - MDB_val_copy<crypto::hash> val_h(tx_hash); - MDB_val unused; - if (mdb_cursor_get(m_cur_txs, &val_h, &unused, MDB_SET) == 0) - throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + txindex ti; + ti.key = tx_hash; + ti.data.tx_id = tx_id; + ti.data.unlock_time = tx.unlock_time; + ti.data.block_id = m_height; // we don't need blk_hash since we know m_height - MDB_val_copy<blobdata> blob(tx_to_blob(tx)); - result = mdb_cursor_put(m_cur_txs, &val_h, &blob, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); + val_h.mv_size = sizeof(ti); + val_h.mv_data = (void *)&ti; - MDB_val_copy<uint64_t> height(m_height); - result = mdb_cursor_put(m_cur_tx_heights, &val_h, &height, 0); + result = mdb_cursor_put(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, 0); if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx block height to db transaction: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); - MDB_val_copy<uint64_t> unlock_time(tx.unlock_time); - result = mdb_cursor_put(m_cur_tx_unlocks, &val_h, &unlock_time, 0); + MDB_val_copy<blobdata> blob(tx_to_blob(tx)); + result = mdb_cursor_put(m_cur_txs, &val_tx_id, &blob, MDB_APPEND); if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx unlock time to db transaction: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); + + m_num_txs++; + return tx_id; } +// TODO: compare pros and cons of looking up the tx hash's tx index once and +// passing it in to functions like this void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) { int result; @@ -682,28 +730,49 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val val_h = {sizeof(tx_hash), (void *)&tx_hash}; - MDB_val unused; - if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(tx_indices) + CURSOR(txs) + CURSOR(tx_outputs) + + MDB_val_set(val_h, tx_hash); + + if (mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + txindex *tip = (txindex *)val_h.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); - if ((result = mdb_del(*m_write_txn, m_txs, &val_h, NULL))) + if ((result = mdb_cursor_get(m_cur_txs, &val_tx_id, NULL, MDB_SET))) + throw1(DB_ERROR(lmdb_error("Failed to locate tx for removal: ", result).c_str())); + result = mdb_cursor_del(m_cur_txs, 0); + if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of tx to db transaction: ", result).c_str())); - if ((result = mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of tx unlock time to db transaction: ", result).c_str())); - if ((result = mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL))) - throw1(DB_ERROR(lmdb_error("Failed to add removal of tx block height to db transaction: ", result).c_str())); - remove_tx_outputs(&val_h, tx); + remove_tx_outputs(tip->data.tx_id, tx); - result = mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL); + result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_id, NULL, MDB_SET); if (result == MDB_NOTFOUND) LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); else if (result) - throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); + throw1(DB_ERROR(lmdb_error("Failed to locate tx outputs for removal: ", result).c_str())); + if (!result) + { + result = mdb_cursor_del(m_cur_tx_outputs, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); + } + + // Don't delete the tx_indices entry until the end, after we're done with val_tx_id + if (mdb_cursor_del(m_cur_tx_indices, 0)) + throw1(DB_ERROR("Failed to add removal of tx index to db transaction")); + + m_num_txs--; } -void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) +uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash, + const tx_out& tx_output, + const uint64_t& local_index, + const uint64_t unlock_time) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -712,187 +781,130 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou int result = 0; CURSOR(output_txs) - CURSOR(tx_outputs) - CURSOR(output_indices) CURSOR(output_amounts) - CURSOR(output_keys) - MDB_val_copy<uint64_t> k(m_num_outputs); - MDB_val_copy<crypto::hash> v(tx_hash); + if (tx_output.target.type() != typeid(txout_to_key)) + throw0(DB_ERROR("Wrong output type: expected txout_to_key")); - result = mdb_cursor_put(m_cur_output_txs, &k, &v, MDB_APPEND); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str())); - result = mdb_cursor_put(m_cur_tx_outputs, &v, &k, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add <tx hash, global output index> to db transaction: ", result).c_str())); + outtx ot = {m_num_outputs, tx_hash, local_index}; + MDB_val_set(vot, ot); - MDB_val_copy<uint64_t> val_local_index(local_index); - result = mdb_cursor_put(m_cur_output_indices, &k, &val_local_index, MDB_APPEND); + result = mdb_cursor_put(m_cur_output_txs, (MDB_val *)&zerokval, &vot, MDB_APPENDDUP); if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx output index to db transaction: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str())); + outkey ok; + MDB_val data; MDB_val_copy<uint64_t> val_amount(tx_output.amount); - result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &k, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to add output amount to db transaction: ", result).c_str())); - - if (tx_output.target.type() == typeid(txout_to_key)) - { - output_data_t od; - od.pubkey = boost::get < txout_to_key > (tx_output.target).key; - od.unlock_time = unlock_time; - od.height = m_height; - - MDB_val_copy<output_data_t> data(od); - //MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key); - if ((result = mdb_cursor_put(m_cur_output_keys, &k, &data, MDB_APPEND))) - throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str())); - } + result = mdb_cursor_get(m_cur_output_amounts, &val_amount, &data, MDB_SET); + if (!result) + { + mdb_size_t num_elems = 0; + result = mdb_cursor_count(m_cur_output_amounts, &num_elems); + if (result) + throw0(DB_ERROR(std::string("Failed to get number of outputs for amount: ").append(mdb_strerror(result)).c_str())); + ok.amount_index = num_elems; + } + else if (result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Failed to get output amount in db transaction: ", result).c_str())); else - { - throw0(DB_ERROR("Wrong output type: expected txout_to_key")); - } + ok.amount_index = 0; + ok.output_id = m_num_outputs; + ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key; + ok.data.unlock_time = unlock_time; + ok.data.height = m_height; + data.mv_data = &ok; + data.mv_size = sizeof(ok); + + if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP))) + throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str())); m_num_outputs++; + return ok.amount_index; } -void BlockchainLMDB::remove_tx_outputs(const MDB_val *tx_hash, const transaction& tx) +void BlockchainLMDB::add_tx_amount_output_indices(const uint64_t tx_id, + const std::vector<uint64_t>& amount_output_indices) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - + check_open(); mdb_txn_cursors *m_cursors = &m_wcursors; - MDB_val v; CURSOR(tx_outputs) - auto result = mdb_cursor_get(m_cur_tx_outputs, (MDB_val *)tx_hash, &v, MDB_SET); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L2("tx has no outputs, so no global output indices"); - } - else if (result) - { - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - } - else - { - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_tx_outputs, &num_elems); + int result = 0; - mdb_cursor_get(m_cur_tx_outputs, (MDB_val *)tx_hash, &v, MDB_LAST_DUP); + int num_outputs = amount_output_indices.size(); - for (uint64_t i = num_elems; i > 0; --i) - { - const tx_out tx_output = tx.vout[i-1]; -#ifndef MISALIGNED_OK - uint64_t tv; - memcpy(&tv, v.mv_data, sizeof(uint64_t)); - v.mv_data = &tv; -#endif - remove_output((const MDB_val *)&v, tx_output.amount); - if (i > 1) - { - mdb_cursor_get(m_cur_tx_outputs, (MDB_val *)tx_hash, &v, MDB_PREV_DUP); - } - } - } -} + MDB_val_set(k_tx_id, tx_id); + MDB_val v; + v.mv_data = (void *)amount_output_indices.data(); + v.mv_size = sizeof(uint64_t) * num_outputs; + // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); -// TODO: probably remove this function -void BlockchainLMDB::remove_output(const tx_out& tx_output) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); - return; + result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_id, &v, MDB_APPEND); + if (result) + throw0(DB_ERROR(std::string("Failed to add <tx hash, amount output index array> to db transaction: ").append(mdb_strerror(result)).c_str())); } -void BlockchainLMDB::remove_output(const MDB_val *out_index, const uint64_t amount) +void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& tx) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - auto result = mdb_del(*m_write_txn, m_output_indices, (MDB_val *)out_index, NULL); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); - } - else if (result) - { - throw1(DB_ERROR(lmdb_error("Error adding removal of output tx index to db transaction", result).c_str())); - } + std::vector<uint64_t> amount_output_indices = get_tx_amount_output_indices(tx_id); - result = mdb_del(*m_write_txn, m_output_txs, (MDB_val *)out_index, NULL); - // if (result != 0 && result != MDB_NOTFOUND) - // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); - } - else if (result) + if (amount_output_indices.empty()) { - throw1(DB_ERROR(lmdb_error("Error adding removal of output tx hash to db transaction", result).c_str())); + if (tx.vout.empty()) + LOG_PRINT_L2("tx has no outputs, so no output indices"); + else + throw0(DB_ERROR("tx has outputs, but no output indices found")); } - result = mdb_del(*m_write_txn, m_output_keys, (MDB_val *)out_index, NULL); - if (result == MDB_NOTFOUND) + for (uint64_t i = tx.vout.size(); i > 0; --i) { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); + const tx_out tx_output = tx.vout[i-1]; + remove_output(tx_output.amount, amount_output_indices[i-1]); } - else if (result) - throw1(DB_ERROR(lmdb_error("Error adding removal of output pubkey to db transaction", result).c_str())); - - remove_amount_output_index(amount, out_index); - - m_num_outputs--; } -void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const MDB_val *global_output_index) +void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(output_amounts); + CURSOR(output_txs); - MDB_val_copy<uint64_t> k(amount); - MDB_val v; + MDB_val_set(k, amount); + MDB_val_set(v, out_index); - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); + auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); - - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP); - - uint64_t amount_output_index = 0; - bool found_index = false; - for (uint64_t i = num_elems; i > 0; --i) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); - if (!memcmp(v.mv_data, global_output_index->mv_data, sizeof(uint64_t))) - { - amount_output_index = i-1; - found_index = true; - break; - } - if (i > 1) - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_DUP); - } - if (found_index) + outkey *ok = (outkey *)v.mv_data; + MDB_val_set(otxk, ok->output_id); + result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &otxk, MDB_GET_BOTH); + if (result == MDB_NOTFOUND) { - // found the amount output index - // now delete it - int result = mdb_cursor_del(m_cur_output_amounts, 0); - if (result) - throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index).append(": ")).c_str(), result).c_str())); + LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); } - else + else if (result) { - // not found - throw1(OUTPUT_DNE("Failed to find amount output index")); + throw1(DB_ERROR(lmdb_error("Error adding removal of output tx to db transaction", result).c_str())); } + result = mdb_cursor_del(m_cur_output_txs, 0); + if (result) + throw0(DB_ERROR(lmdb_error(std::string("Error deleting output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str())); + + // now delete the amount + result = mdb_cursor_del(m_cur_output_amounts, 0); + if (result) + throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str())); + + m_num_outputs--; } void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) @@ -903,27 +915,33 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) CURSOR(spent_keys) - MDB_val_copy<crypto::key_image> val_key(k_image); - MDB_val unused; - if (mdb_cursor_get(m_cur_spent_keys, &val_key, &unused, MDB_SET) == 0) + MDB_val k = {sizeof(k_image), (void *)&k_image}; + if (auto result = mdb_cursor_put(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_NODUPDATA)) { + if (result == MDB_KEYEXIST) throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); - - char anything = '\0'; - unused.mv_size = sizeof(char); - unused.mv_data = &anything; - if (auto result = mdb_cursor_put(m_cur_spent_keys, &val_key, &unused, 0)) - throw1(DB_ERROR(lmdb_error("Error adding spent key image to db transaction: ", result).c_str())); + else + throw1(DB_ERROR(lmdb_error("Error adding spent key image to db transaction: ", result).c_str())); + } } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + + CURSOR(spent_keys) - MDB_val_copy<crypto::key_image> k(k_image); - auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); + MDB_val k = {sizeof(k_image), (void *)&k_image}; + auto result = mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH); if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str())); + throw1(DB_ERROR(lmdb_error("Error finding spent key to remove", result).c_str())); + if (!result) + { + result = mdb_cursor_del(m_cur_spent_keys, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str())); + } } blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const @@ -949,19 +967,6 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const return o; } -uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::vector <uint64_t> offsets; - std::vector <uint64_t> global_indices; - offsets.push_back(index); - get_output_global_indices(amount, offsets, global_indices); - if (!global_indices.size()) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - - return global_indices[0]; -} - void BlockchainLMDB::check_open() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1074,39 +1079,31 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // uses macros to avoid having to change things too many places lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); - lmdb_db_open(txn, LMDB_BLOCK_TIMESTAMPS, MDB_INTEGERKEY | MDB_CREATE, m_block_timestamps, "Failed to open db handle for m_block_timestamps"); - lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); - lmdb_db_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, m_block_hashes, "Failed to open db handle for m_block_hashes"); - lmdb_db_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, m_block_sizes, "Failed to open db handle for m_block_sizes"); - lmdb_db_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, m_block_diffs, "Failed to open db handle for m_block_diffs"); - lmdb_db_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, m_block_coins, "Failed to open db handle for m_block_coins"); + lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info"); + lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for m_block_heights"); - lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); - lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); - lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); - lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); + lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); - lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); - lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); - lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); - lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); lmdb_db_open(txn, LMDB_HF_STARTING_HEIGHTS, MDB_CREATE, m_hf_starting_heights, "Failed to open db handle for m_hf_starting_heights"); - lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions"); + lmdb_db_open(txn, LMDB_HF_VERSIONS, MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for m_hf_versions"); lmdb_db_open(txn, LMDB_PROPERTIES, MDB_CREATE, m_properties, "Failed to open db handle for m_properties"); + mdb_set_dupsort(txn, m_spent_keys, compare_hash32); + mdb_set_dupsort(txn, m_block_heights, compare_hash32); + mdb_set_dupsort(txn, m_tx_indices, compare_hash32); mdb_set_dupsort(txn, m_output_amounts, compare_uint64); - mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); - mdb_set_compare(txn, m_spent_keys, compare_hash32); - mdb_set_compare(txn, m_block_heights, compare_hash32); - mdb_set_compare(txn, m_txs, compare_hash32); - mdb_set_compare(txn, m_tx_unlocks, compare_hash32); - mdb_set_compare(txn, m_tx_heights, compare_hash32); + mdb_set_dupsort(txn, m_output_txs, compare_uint64); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); - mdb_set_compare(txn, m_hf_versions, compare_uint64); mdb_set_compare(txn, m_properties, compare_string); // get and keep current height @@ -1116,34 +1113,18 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); m_height = db_stats.ms_entries; + // get and keep current number of txs + if ((result = mdb_stat(txn, m_txs, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str())); + m_num_txs = db_stats.ms_entries; + // get and keep current number of outputs - if ((result = mdb_stat(txn, m_output_indices, &db_stats))) - throw0(DB_ERROR(lmdb_error("Failed to query m_output_indices: ", result).c_str())); + if ((result = mdb_stat(txn, m_output_txs, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str())); m_num_outputs = db_stats.ms_entries; bool compatible = true; - // ND: This "new" version of the lmdb database is incompatible with - // the previous version. Ensure that the output_keys database is - // sizeof(output_data_t) in length. Otherwise, inform user and - // terminate. - if(m_height > 0) - { - MDB_val_copy<uint64_t> k(0); - MDB_val v; - auto get_result = mdb_get(txn, m_output_keys, &k, &v); - if(get_result != MDB_SUCCESS) - { - txn.abort(); - m_open = false; - return; - } - - // LOG_PRINT_L0("Output keys size: " << v.mv_size); - if(v.mv_size != sizeof(output_data_t)) - compatible = false; - } - MDB_val_copy<const char*> k("version"); MDB_val v; auto get_result = mdb_get(txn, m_properties, &k, &v); @@ -1157,13 +1138,21 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) #if VERSION > 0 else if (*(const uint32_t*)v.mv_data < VERSION) { - compatible = false; + // Note that there was a schema change within version 0 as well. + // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 + // We don't handle the old format previous to that commit. + txn.commit(); + m_open = true; + migrate(*(const uint32_t *)v.mv_data); + return; } #endif } else { - // if not found, but we're on version 0, it's fine. If the DB's empty, it's fine too. + // if not found, and the DB is non-empty, this is probably + // an "old" version 0, which we don't handle. If the DB is + // empty it's fine. if (VERSION > 0 && m_height > 0) compatible = false; } @@ -1242,20 +1231,12 @@ void BlockchainLMDB::reset() if (auto result = mdb_txn_begin(m_env, NULL, 0, txn)) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); mdb_drop(txn, m_blocks, 0); - mdb_drop(txn, m_block_timestamps, 0); + mdb_drop(txn, m_block_info, 0); mdb_drop(txn, m_block_heights, 0); - mdb_drop(txn, m_block_hashes, 0); - mdb_drop(txn, m_block_sizes, 0); - mdb_drop(txn, m_block_diffs, 0); - mdb_drop(txn, m_block_coins, 0); mdb_drop(txn, m_txs, 0); - mdb_drop(txn, m_tx_unlocks, 0); - mdb_drop(txn, m_tx_heights, 0); mdb_drop(txn, m_tx_outputs, 0); mdb_drop(txn, m_output_txs, 0); - mdb_drop(txn, m_output_indices, 0); mdb_drop(txn, m_output_amounts, 0); - mdb_drop(txn, m_output_keys, 0); mdb_drop(txn, m_spent_keys, 0); mdb_drop(txn, m_hf_starting_heights, 0); mdb_drop(txn, m_hf_versions, 0); @@ -1365,8 +1346,8 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const RCURSOR(block_heights); bool ret = false; - MDB_val_copy<crypto::hash> key(h); - auto get_result = mdb_cursor_get(m_cur_block_heights, &key, NULL, MDB_SET); + MDB_val_set(key, h); + auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); @@ -1396,15 +1377,15 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const TXN_PREFIX_RDONLY(); RCURSOR(block_heights); - MDB_val_copy<crypto::hash> key(h); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_heights, &key, &result, MDB_SET); + MDB_val_set(key, h); + auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); - uint64_t ret = *(const uint64_t *)result.mv_data; + blk_height *bhp = (blk_height *)key.mv_data; + uint64_t ret = bhp->bh_height; TXN_POSTFIX_RDONLY(); return ret; } @@ -1454,11 +1435,10 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_timestamps); + RCURSOR(block_info); - MDB_val_copy<uint64_t> key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_timestamps, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str())); @@ -1466,7 +1446,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); - uint64_t ret = *(const uint64_t *)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_timestamp; TXN_POSTFIX_RDONLY(); return ret; } @@ -1491,11 +1472,10 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_sizes); + RCURSOR(block_info); - MDB_val_copy<uint64_t> key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_sizes, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); @@ -1503,7 +1483,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); - size_t ret = *(const size_t *)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + size_t ret = bi->bi_size; TXN_POSTFIX_RDONLY(); return ret; } @@ -1514,11 +1495,10 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_diffs); + RCURSOR(block_info); - MDB_val_copy<uint64_t> key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_diffs, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str())); @@ -1526,7 +1506,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - difficulty_type ret = *(const difficulty_type*)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + difficulty_type ret = bi->bi_diff; TXN_POSTFIX_RDONLY(); return ret; } @@ -1554,11 +1535,10 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_coins); + RCURSOR(block_info); - MDB_val_copy<uint64_t> key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_coins, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); @@ -1566,7 +1546,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); - uint64_t ret = *(const uint64_t*)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_coins; TXN_POSTFIX_RDONLY(); return ret; } @@ -1577,11 +1558,10 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(block_hashes); + RCURSOR(block_info); - MDB_val_copy<uint64_t> key(height); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_block_hashes, &key, &result, MDB_SET); + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str())); @@ -1589,7 +1569,8 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) else if (get_result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a block hash from the db: ", get_result).c_str())); - crypto::hash ret = *(const crypto::hash*)result.mv_data; + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + crypto::hash ret = bi->bi_hash; TXN_POSTFIX_RDONLY(); return ret; } @@ -1662,15 +1643,58 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); RCURSOR(txs); - MDB_val_copy<crypto::hash> key(h); - MDB_val result; + MDB_val_set(key, h); + bool tx_found = false; + + TIME_MEASURE_START(time1); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); + if (get_result == 0) + tx_found = true; + else if (get_result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", get_result).c_str())); + + // This isn't needed as part of the check. we're not checking consistency of db. + // get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + + TXN_POSTFIX_RDONLY(); + + if (! tx_found) + { + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + + // Below not needed due to above comment. + // if (get_result == MDB_NOTFOUND) + // throw0(DB_ERROR(std::string("transaction with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found at index").c_str())); + // else if (get_result) + // throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction ") + epee::string_tools::pod_to_hex(h) + " at index: ", get_result).c_str())); + return true; +} + +bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_id) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + + MDB_val_set(v, h); TIME_MEASURE_START(time1); - auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; + if (!get_result) { + txindex *tip = (txindex *)v.mv_data; + tx_id = tip->data.tx_id; + } TXN_POSTFIX_RDONLY(); @@ -1693,17 +1717,17 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(tx_unlocks); + RCURSOR(tx_indices); - MDB_val_copy<crypto::hash> key(h); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_unlocks, &key, &result, MDB_SET); + MDB_val_set(v, h); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + throw1(TX_DNE(lmdb_error(std::string("tx data with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); else if (get_result) - throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx unlock time from hash", get_result).c_str())); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx data from hash: ", get_result).c_str())); - uint64_t ret = *(const uint64_t*)result.mv_data; + txindex *tip = (txindex *)v.mv_data; + uint64_t ret = tip->data.unlock_time; TXN_POSTFIX_RDONLY(); return ret; } @@ -1714,11 +1738,18 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); RCURSOR(txs); - MDB_val_copy<crypto::hash> key(h); + MDB_val_set(v, h); MDB_val result; - auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == 0) + { + txindex *tip = (txindex *)v.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs, &val_tx_id, &result, MDB_SET); + } if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) @@ -1744,8 +1775,8 @@ uint64_t BlockchainLMDB::get_tx_count() const TXN_PREFIX_RDONLY(); MDB_stat db_stats; - if (mdb_stat(m_txn, m_txs, &db_stats)) - throw0(DB_ERROR("Failed to query m_txs")); + if (mdb_stat(m_txn, m_tx_indices, &db_stats)) + throw0(DB_ERROR("Failed to query m_tx_indices")); TXN_POSTFIX_RDONLY(); @@ -1772,19 +1803,19 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(tx_heights); + RCURSOR(tx_indices); - MDB_val_copy<crypto::hash> key(h); - MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_heights, &key, &result, MDB_SET); + MDB_val_set(v, h); + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) { - throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + throw1(TX_DNE(std::string("tx_data_t with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); } else if (get_result) throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx height from hash", get_result).c_str())); - uint64_t ret = *(const uint64_t*)result.mv_data; + txindex *tip = (txindex *)v.mv_data; + uint64_t ret = tip->data.block_id; TXN_POSTFIX_RDONLY(); return ret; } @@ -1813,24 +1844,54 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } +// This is a lot harder now that we've removed the output_keys index output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); check_open(); - TXN_PREFIX_RDONLY(); - RCURSOR(output_keys); + RCURSOR(output_txs); + RCURSOR(tx_indices); + RCURSOR(txs); - MDB_val_copy<uint64_t> k(global_index); - MDB_val v; - auto get_result = mdb_cursor_get(m_cur_output_keys, &k, &v, MDB_SET); + output_data_t od; + MDB_val_set(v, global_index); + auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); + throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); - output_data_t ret = *(const output_data_t *) v.mv_data; + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + outtx *ot = (outtx *)v.mv_data; + + MDB_val_set(val_h, ot->tx_hash); + get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); + if (get_result) + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(ot->tx_hash) + ": ", get_result).c_str())); + + txindex *tip = (txindex *)val_h.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + MDB_val result; + get_result = mdb_cursor_get(m_cur_txs, &val_tx_id, &result, MDB_SET); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(ot->tx_hash)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + + blobdata bd; + bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + + const tx_out tx_output = tx.vout[ot->local_index]; + od.unlock_time = tip->data.unlock_time; + od.height = tip->data.block_id; + od.pubkey = boost::get<txout_to_key>(tx_output.target).key; + TXN_POSTFIX_RDONLY(); - return ret; + return od; } output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) @@ -1838,37 +1899,41 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - uint64_t glob_index = get_output_global_index(amount, index); - return get_output_key(glob_index); + TXN_PREFIX_RDONLY(); + RCURSOR(output_amounts); + + MDB_val_set(k, amount); + MDB_val_set(v, index); + auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get output pubkey by index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + outkey *okp = (outkey *)v.mv_data; + output_data_t ret = okp->data; + TXN_POSTFIX_RDONLY(); + return ret; } -tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const +tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& output_id) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); TXN_PREFIX_RDONLY(); RCURSOR(output_txs); - RCURSOR(output_indices); - MDB_val_copy<uint64_t> k(index); - MDB_val v; + MDB_val_set(v, output_id); - auto get_result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - crypto::hash tx_hash = *(const crypto::hash*)v.mv_data; + outtx *ot = (outtx *)v.mv_data; + tx_out_index ret = tx_out_index(ot->tx_hash, ot->local_index); - get_result = mdb_cursor_get(m_cur_output_indices, &k, &v, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx index")); - - tx_out_index ret = tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); TXN_POSTFIX_RDONLY(); return ret; } @@ -1886,128 +1951,55 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } -std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const +std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_id) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); - std::vector<uint64_t> index_vec; TXN_PREFIX_RDONLY(); RCURSOR(tx_outputs); - MDB_val_copy<crypto::hash> k(h); + int result = 0; + MDB_val_set(k_tx_id, tx_id); MDB_val v; - auto result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET); + std::vector<uint64_t> amount_output_indices; + + result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, MDB_SET); if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " + "tx_outputs, but it should have an empty entry even if it's a tx without " + "outputs"); else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); + throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_tx_outputs, &num_elems); + uint64_t* indices = (uint64_t*)v.mv_data; + int num_outputs = v.mv_size / sizeof(uint64_t); - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < num_elems; ++i) + for (int i = 0; i < num_outputs; ++i) { - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_GET_CURRENT); - index_vec.push_back(*(const uint64_t *)v.mv_data); - mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_NEXT_DUP); + // LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]); + amount_output_indices.push_back(indices[i]); } + indices = nullptr; TXN_POSTFIX_RDONLY(); - - return index_vec; + return amount_output_indices; } -std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - std::vector<uint64_t> index_vec; - std::vector<uint64_t> index_vec2; - - // get the transaction's global output indices first - index_vec = get_tx_output_indices(h); - // these are next used to obtain the amount output indices - - transaction tx = get_tx(h); - - TXN_PREFIX_RDONLY(); - RCURSOR(output_amounts); - - uint64_t i = 0; - uint64_t global_index; - BOOST_FOREACH(const auto& vout, tx.vout) - { - uint64_t amount = vout.amount; - - global_index = index_vec[i]; - - MDB_val_copy<uint64_t> k(amount); - MDB_val v; - - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); - - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_FIRST_DUP); - - uint64_t amount_output_index = 0; - uint64_t output_index = 0; - bool found_index = false; - for (uint64_t j = 0; j < num_elems; ++j) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); - output_index = *(const uint64_t *)v.mv_data; - if (output_index == global_index) - { - amount_output_index = j; - found_index = true; - break; - } - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); - } - if (found_index) - { - index_vec2.push_back(amount_output_index); - } - else - { - // not found - TXN_POSTFIX_RDONLY(); - throw1(OUTPUT_DNE("specified output not found in db")); - } - - ++i; - } - - TXN_POSTFIX_RDONLY(); - - return index_vec2; -} - - bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + bool ret; + TXN_PREFIX_RDONLY(); RCURSOR(spent_keys); - bool ret = false; - MDB_val_copy<crypto::key_image> val_key(img); - if (mdb_cursor_get(m_cur_spent_keys, &val_key, NULL, MDB_SET) == 0) - { - ret = true; - } + MDB_val k = {sizeof(img), (void *)&img}; + ret = (mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH) == 0); TXN_POSTFIX_RDONLY(); return ret; @@ -2021,10 +2013,10 @@ bool BlockchainLMDB::for_all_key_images(std::function<bool(const crypto::key_ima TXN_PREFIX_RDONLY(); RCURSOR(spent_keys); - MDB_val k; - MDB_val v; + MDB_val k, v; bool ret = true; + k = zerokval; MDB_cursor_op op = MDB_FIRST; while (1) { @@ -2034,7 +2026,7 @@ bool BlockchainLMDB::for_all_key_images(std::function<bool(const crypto::key_ima break; if (ret < 0) throw0(DB_ERROR("Failed to enumerate key images")); - const crypto::key_image k_image = *(const crypto::key_image*)k.mv_data; + const crypto::key_image k_image = *(const crypto::key_image*)v.mv_data; if (!f(k_image)) { ret = false; break; @@ -2094,6 +2086,7 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash& TXN_PREFIX_RDONLY(); RCURSOR(txs); + RCURSOR(tx_indices); MDB_val k; MDB_val v; @@ -2102,13 +2095,22 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash& MDB_cursor_op op = MDB_FIRST; while (1) { - int ret = mdb_cursor_get(m_cur_txs, &k, &v, op); + int ret = mdb_cursor_get(m_cur_tx_indices, &k, &v, op); op = MDB_NEXT; if (ret == MDB_NOTFOUND) break; if (ret) - throw0(DB_ERROR("Failed to enumerate transactions")); - const crypto::hash hash = *(const crypto::hash*)k.mv_data; + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + txindex *ti = (txindex *)v.mv_data; + const crypto::hash hash = ti->key; + k.mv_data = (void *)&ti->data.tx_id; + k.mv_size = sizeof(ti->data.tx_id); + ret = mdb_cursor_get(m_cur_txs, &k, &v, MDB_SET); + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); blobdata bd; bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); transaction tx; @@ -2147,8 +2149,8 @@ bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const c if (ret) throw0(DB_ERROR("Failed to enumerate outputs")); uint64_t amount = *(const uint64_t*)k.mv_data; - uint64_t global_index = *(const uint64_t*)v.mv_data; - tx_out_index toi = get_output_tx_and_index_from_global(global_index); + outkey *ok = (outkey *)v.mv_data; + tx_out_index toi = get_output_tx_and_index_from_global(ok->output_id); if (!f(amount, toi.first, toi.second)) { ret = false; break; @@ -2267,6 +2269,10 @@ void BlockchainLMDB::batch_abort() void BlockchainLMDB::set_batch_transactions(bool batch_transactions) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if ((batch_transactions) && (m_batch_transactions)) + { + LOG_PRINT_L0("WARNING: batch transaction mode already enabled, but asked to enable batch mode"); + } m_batch_transactions = batch_transactions; LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } @@ -2431,6 +2437,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } } + uint64_t num_txs = m_num_txs; uint64_t num_outputs = m_num_outputs; try { @@ -2442,6 +2449,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } catch (...) { + m_num_txs = num_txs; m_num_outputs = num_outputs; block_txn_abort(); throw; @@ -2457,6 +2465,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) block_txn_start(false); + uint64_t num_txs = m_num_txs; uint64_t num_outputs = m_num_outputs; try { @@ -2465,6 +2474,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) } catch (...) { + m_num_txs = num_txs; m_num_outputs = num_outputs; block_txn_abort(); throw; @@ -2482,211 +2492,88 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6 TXN_PREFIX_RDONLY(); RCURSOR(output_txs); - RCURSOR(output_indices); - for (const uint64_t &index : global_indices) + for (const uint64_t &output_id : global_indices) { - MDB_val_copy<uint64_t> k(index); - MDB_val v; + MDB_val_set(v, output_id); - auto get_result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - crypto::hash tx_hash = *(const crypto::hash*) v.mv_data; - - get_result = mdb_cursor_get(m_cur_output_indices, &k, &v, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx index")); - - auto result = tx_out_index(tx_hash, *(const uint64_t *) v.mv_data); + outtx *ot = (outtx *)v.mv_data; + auto result = tx_out_index(ot->tx_hash, ot->local_index); tx_out_indices.push_back(result); } TXN_POSTFIX_RDONLY(); } -void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, - std::vector<uint64_t> &global_indices) +void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - TIME_MEASURE_START(txx); + TIME_MEASURE_START(db3); check_open(); - global_indices.clear(); - - uint64_t max = 0; - for (const uint64_t &index : offsets) - { - if (index > max) - max = index; - } + outputs.clear(); TXN_PREFIX_RDONLY(); - RCURSOR(output_amounts); - - MDB_val_copy<uint64_t> k(amount); - MDB_val v; - auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - - mdb_size_t num_elems = 0; - mdb_cursor_count(m_cur_output_amounts, &num_elems); - if (max <= 1 && num_elems <= max) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - uint64_t t_dbmul = 0; - uint64_t t_dbscan = 0; - if (max <= 1) - { - for (const uint64_t& index : offsets) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_FIRST_DUP); - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); - } + RCURSOR(output_amounts); - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); - uint64_t glob_index = *(const uint64_t*) v.mv_data; - LOG_PRINT_L3("Amount: " << amount << " M0->v: " << glob_index); - global_indices.push_back(glob_index); - } - } - else + MDB_val_set(k, amount); + for (const uint64_t &index : offsets) { - uint32_t curcount = 0; - uint32_t blockstart = 0; - for (const uint64_t& index : offsets) - { - if (index >= num_elems) - { - LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index"); - break; - } - if (!curcount && index > num_elems/2) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP); - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV); /* kludge to unset C_EOF */ - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT); - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_MULTIPLE); - - curcount = num_elems; - while(1) - { - TIME_MEASURE_START(db1); - int count = v.mv_size / sizeof(uint64_t); - curcount -= count; - if (curcount > index) - { - mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_MULTIPLE); - } else - { - blockstart = curcount; - curcount += count; - break; - } - TIME_MEASURE_FINISH(db1); - t_dbmul += db1; - } - - } else - { - while (index >= curcount) - { - TIME_MEASURE_START(db1); - if (mdb_cursor_get(m_cur_output_amounts, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0) - { - // allow partial results - result = false; - break; - } + MDB_val_set(v, index); - int count = v.mv_size / sizeof(uint64_t); - - blockstart = curcount; - curcount += count; - TIME_MEASURE_FINISH(db1); - t_dbmul += db1; - } - } - - LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index); - TIME_MEASURE_START(db2); - uint64_t actual_index = index - blockstart; - uint64_t glob_index = ((const uint64_t*) v.mv_data)[actual_index]; - - LOG_PRINT_L3("Amount: " << amount << " M1->v: " << glob_index); - global_indices.push_back(glob_index); - - TIME_MEASURE_FINISH(db2); - t_dbscan += db2; + auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); - } + outkey *okp = (outkey *)v.mv_data; + output_data_t data = okp->data; + outputs.push_back(data); } TXN_POSTFIX_RDONLY(); - TIME_MEASURE_FINISH(txx); - LOG_PRINT_L3("txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan); + TIME_MEASURE_FINISH(db3); + LOG_PRINT_L3("db3: " << db3); } -void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) +void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - TIME_MEASURE_START(db3); check_open(); - outputs.clear(); + indices.clear(); + std::vector <uint64_t> tx_indices; TXN_PREFIX_RDONLY(); - std::vector <uint64_t> global_indices; - get_output_global_indices(amount, offsets, global_indices); - - if (global_indices.size() > 0) - { - RCURSOR(output_keys); - for (const uint64_t &index : global_indices) - { - MDB_val_copy<uint64_t> k(index); - MDB_val v; + RCURSOR(output_amounts); - auto get_result = mdb_cursor_get(m_cur_output_keys, &k, &v, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); - else if (get_result) - throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); + MDB_val_set(k, amount); + for (const uint64_t &index : offsets) + { + MDB_val_set(v, index); - output_data_t data = *(const output_data_t *) v.mv_data; - outputs.push_back(data); - } + auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get output by index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output from the db", get_result).c_str())); + outkey *okp = (outkey *)v.mv_data; + tx_indices.push_back(okp->output_id); } - TXN_POSTFIX_RDONLY(); - - TIME_MEASURE_FINISH(db3); - LOG_PRINT_L3("db3: " << db3); -} - -void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - indices.clear(); - - std::vector <uint64_t> global_indices; - get_output_global_indices(amount, offsets, global_indices); TIME_MEASURE_START(db3); - if(global_indices.size() > 0) + if(tx_indices.size() > 0) { - get_output_tx_and_index_from_global(global_indices, indices); + get_output_tx_and_index_from_global(tx_indices, indices); } TIME_MEASURE_FINISH(db3); LOG_PRINT_L3("db3: " << db3); @@ -2813,7 +2700,7 @@ uint64_t BlockchainLMDB::get_hard_fork_starting_height(uint8_t version) const MDB_val_copy<uint8_t> val_key(version); MDB_val val_ret; - uint64_t ret; + uint64_t ret = 0; auto result = mdb_get(m_txn, m_hf_starting_heights, &val_key, &val_ret); if (result == MDB_SUCCESS) { @@ -2890,4 +2777,558 @@ void BlockchainLMDB::fixup() BlockchainDB::fixup(); } +#define RENAME_DB(name) \ + k.mv_data = (void *)name; \ + k.mv_size = sizeof(name)-1; \ + result = mdb_cursor_open(txn, 1, &c_cur); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for " name ": ", result).c_str())); \ + result = mdb_cursor_get(c_cur, &k, NULL, MDB_SET_KEY); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \ + ptr = (char *)k.mv_data; \ + ptr[sizeof(name)-2] = 's' + +#define LOGIF(y) if (y <= epee::log_space::log_singletone::get_log_detalisation_level()) + +void BlockchainLMDB::migrate_0_1() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i, z; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + LOG_PRINT_YELLOW("Migrating blockchain from DB version 0 to 1 - this may take a while:", LOG_LEVEL_0); + LOG_PRINT_L0("updating blocks, hf_versions, outputs, txs, and spent_keys tables..."); + + LOG_PRINT_L0("Total number of blocks: " << m_height); + LOG_PRINT_L1("block migration will update block_heights, block_info, and hf_versions..."); + + do { + LOG_PRINT_L1("migrating block_heights:"); + MDB_dbi o_heights; + + unsigned int flags; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_flags(txn, m_block_heights, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve block_heights flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if ((flags & (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) == (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) { + txn.abort(); + LOG_PRINT_L1(" block_heights already migrated"); + break; + } + + /* the block_heights table name is the same but the old version and new version + * have incompatible DB flags. Create a new table with the right flags. We want + * the name to be similar to the old name so that it will occupy the same location + * in the DB. + */ + o_heights = m_block_heights; + lmdb_db_open(txn, "block_heightr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for block_heightr"); + mdb_set_dupsort(txn, m_block_heights, compare_hash32); + + MDB_cursor *c_old, *c_cur; + blk_height bh; + MDB_val_set(nv, bh); + + /* old table was k(hash), v(height). + * new table is DUPFIXED, k(zeroval), v{hash, height}. + */ + i = 0; + z = m_height; + while(1) { + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_heights, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heightr: ", result).c_str())); + result = mdb_cursor_open(txn, o_heights, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heights: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_block_heights, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_heights: ", result).c_str())); + bh.bh_hash = *(crypto::hash *)k.mv_data; + bh.bh_height = *(uint64_t *)v.mv_data; + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_heightr: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_heights: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_heights, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_heights table: ", result).c_str())); + + RENAME_DB("block_heightr"); + + /* close and reopen to get old dbi slot back */ + mdb_dbi_close(m_env, m_block_heights); + lmdb_db_open(txn, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for block_heights"); + mdb_set_dupsort(txn, m_block_heights, compare_hash32); + txn.commit(); + + } while(0); + + /* old tables are k(height), v(value). + * new table is DUPFIXED, k(zeroval), v{height, values...}. + */ + do { + LOG_PRINT_L1("migrating block info:"); + + MDB_dbi coins; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_open(txn, "block_coins", 0, &coins); + if (result == MDB_NOTFOUND) { + txn.abort(); + LOG_PRINT_L1(" block_info already migrated"); + break; + } + MDB_dbi diffs, hashes, sizes, timestamps; + mdb_block_info bi; + MDB_val_set(nv, bi); + + lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs"); + lmdb_db_open(txn, "block_hashes", 0, hashes, "Failed to open db handle for block_hashes"); + lmdb_db_open(txn, "block_sizes", 0, sizes, "Failed to open db handle for block_sizes"); + lmdb_db_open(txn, "block_timestamps", 0, timestamps, "Failed to open db handle for block_timestamps"); + MDB_cursor *c_cur, *c_coins, *c_diffs, *c_hashes, *c_sizes, *c_timestamps; + i = 0; + z = m_height; + while(1) { + MDB_val k, v; + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + result = mdb_cursor_open(txn, coins, &c_coins); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_coins: ", result).c_str())); + result = mdb_cursor_open(txn, diffs, &c_diffs); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_diffs: ", result).c_str())); + result = mdb_cursor_open(txn, hashes, &c_hashes); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_hashes: ", result).c_str())); + result = mdb_cursor_open(txn, sizes, &c_sizes); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_coins: ", result).c_str())); + result = mdb_cursor_open(txn, timestamps, &c_timestamps); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_timestamps: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_block_info, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_coins, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + break; + } else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_coins: ", result).c_str())); + bi.bi_height = *(uint64_t *)k.mv_data; + bi.bi_coins = *(uint64_t *)v.mv_data; + result = mdb_cursor_get(c_diffs, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_diffs: ", result).c_str())); + bi.bi_diff = *(uint64_t *)v.mv_data; + result = mdb_cursor_get(c_hashes, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_hashes: ", result).c_str())); + bi.bi_hash = *(crypto::hash *)v.mv_data; + result = mdb_cursor_get(c_sizes, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str())); + if (v.mv_size == sizeof(uint32_t)) + bi.bi_size = *(uint32_t *)v.mv_data; + else + bi.bi_size = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 + result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str())); + bi.bi_timestamp = *(uint64_t *)v.mv_data; + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_info: ", result).c_str())); + result = mdb_cursor_del(c_coins, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_coins: ", result).c_str())); + result = mdb_cursor_del(c_diffs, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_diffs: ", result).c_str())); + result = mdb_cursor_del(c_hashes, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_hashes: ", result).c_str())); + result = mdb_cursor_del(c_sizes, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_sizes: ", result).c_str())); + result = mdb_cursor_del(c_timestamps, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_timestamps: ", result).c_str())); + i++; + } + mdb_cursor_close(c_timestamps); + mdb_cursor_close(c_sizes); + mdb_cursor_close(c_hashes); + mdb_cursor_close(c_diffs); + mdb_cursor_close(c_coins); + result = mdb_drop(txn, timestamps, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_timestamps from the db: ", result).c_str())); + result = mdb_drop(txn, sizes, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_sizes from the db: ", result).c_str())); + result = mdb_drop(txn, hashes, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_hashes from the db: ", result).c_str())); + result = mdb_drop(txn, diffs, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_diffs from the db: ", result).c_str())); + result = mdb_drop(txn, coins, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete block_coins from the db: ", result).c_str())); + txn.commit(); + } while(0); + + do { + LOG_PRINT_L1("migrating hf_versions:"); + MDB_dbi o_hfv; + + unsigned int flags; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_flags(txn, m_hf_versions, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve hf_versions flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if (flags & MDB_INTEGERKEY) { + txn.abort(); + LOG_PRINT_L1(" hf_versions already migrated"); + break; + } + + /* the hf_versions table name is the same but the old version and new version + * have incompatible DB flags. Create a new table with the right flags. + */ + o_hfv = m_hf_versions; + lmdb_db_open(txn, "hf_versionr", MDB_INTEGERKEY | MDB_CREATE, m_hf_versions, "Failed to open db handle for hf_versionr"); + + MDB_cursor *c_old, *c_cur; + i = 0; + z = m_height; + + while(1) { + if (!(i % 2000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_hf_versions, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keyr: ", result).c_str())); + result = mdb_cursor_open(txn, o_hfv, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keys: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_hf_versions, &ms); + i = ms.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from hf_versions: ", result).c_str())); + result = mdb_cursor_put(c_cur, &k, &v, MDB_APPEND); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into hf_versionr: ", result).c_str())); + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from hf_versions: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_hfv, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old hf_versions table: ", result).c_str())); + RENAME_DB("hf_versionr"); + mdb_dbi_close(m_env, m_hf_versions); + lmdb_db_open(txn, "hf_versions", MDB_INTEGERKEY, m_hf_versions, "Failed to open db handle for hf_versions"); + + txn.commit(); + } while(0); + + do { + LOG_PRINT_L1("deleting old indices:"); + + /* Delete all other tables, we're just going to recreate them */ + MDB_dbi dbi; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + result = mdb_dbi_open(txn, "tx_unlocks", 0, &dbi); + if (result == MDB_NOTFOUND) { + txn.abort(); + LOG_PRINT_L1(" old indices already deleted"); + break; + } + txn.abort(); + +#define DELETE_DB(x) do { \ + LOG_PRINT_L1(" " x ":"); \ + result = mdb_txn_begin(m_env, NULL, 0, txn); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); \ + result = mdb_dbi_open(txn, x, 0, &dbi); \ + if (!result) { \ + result = mdb_drop(txn, dbi, 1); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to delete " x ": ", result).c_str())); \ + txn.commit(); \ + } } while(0) + + DELETE_DB("tx_heights"); + DELETE_DB("output_txs"); + DELETE_DB("output_indices"); + DELETE_DB("output_keys"); + DELETE_DB("spent_keys"); + DELETE_DB("output_amounts"); + DELETE_DB("tx_outputs"); + DELETE_DB("tx_unlocks"); + + /* reopen new DBs with correct flags */ + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for m_output_txs"); + mdb_set_dupsort(txn, m_output_txs, compare_uint64); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); + mdb_set_dupsort(txn, m_spent_keys, compare_hash32); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); + txn.commit(); + m_num_txs = 0; + m_num_outputs = 0; + } while(0); + + do { + LOG_PRINT_L1("migrating txs and outputs:"); + + unsigned int flags; + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_dbi_flags(txn, m_txs, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve txs flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if (flags & MDB_INTEGERKEY) { + txn.abort(); + LOG_PRINT_L1(" txs already migrated"); + break; + } + + MDB_dbi o_txs; + blobdata bd; + block b; + MDB_val hk; + + o_txs = m_txs; + mdb_set_compare(txn, o_txs, compare_hash32); + lmdb_db_open(txn, "txr", MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for txr"); + + txn.commit(); + + MDB_cursor *c_blocks, *c_txs, *c_props, *c_cur; + i = 0; + z = m_height; + + hk.mv_size = sizeof(crypto::hash); + set_batch_transactions(true); + batch_start(1000); + txn.m_txn = m_write_txn->m_txn; + m_height = 0; + + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(1) { + std::cout << i << " / " << z << " \r" << std::flush; + } + MDB_val_set(pk, "txblk"); + MDB_val_set(pv, m_height); + result = mdb_cursor_put(c_props, &pk, &pv, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update txblk property: ", result).c_str())); + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + m_write_txn->m_txn = txn.m_txn; + m_write_batch_txn->m_txn = txn.m_txn; + memset(&m_wcursors, 0, sizeof(m_wcursors)); + } + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + result = mdb_cursor_open(txn, m_properties, &c_props); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for properties: ", result).c_str())); + result = mdb_cursor_open(txn, o_txs, &c_txs); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str())); + if (!i) { + MDB_stat ms; + mdb_stat(txn, m_output_txs, &ms); + m_num_outputs = ms.ms_entries; + mdb_stat(txn, m_txs, &ms); + m_num_txs = i = ms.ms_entries; + if (i) { + m_num_txs = i; + MDB_val_set(pk, "txblk"); + result = mdb_cursor_get(c_props, &pk, &k, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from properties: ", result).c_str())); + m_height = *(uint64_t *)k.mv_data; + } + } + if (i) { + result = mdb_cursor_get(c_blocks, &k, &v, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str())); + } + } + result = mdb_cursor_get(c_blocks, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + MDB_val_set(pk, "txblk"); + mdb_cursor_get(c_props, &pk, &v, MDB_SET); + mdb_cursor_del(c_props, 0); + batch_stop(); + break; + } else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str())); + + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + add_transaction(null_hash, b.miner_tx); + for (unsigned int j = 0; j<b.tx_hashes.size(); j++) { + transaction tx; + hk.mv_data = &b.tx_hashes[j]; + result = mdb_cursor_get(c_txs, &hk, &v, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + add_transaction(null_hash, tx, &b.tx_hashes[j]); + result = mdb_cursor_del(c_txs, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); + } + i++; + m_height = i; + } + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_drop(txn, o_txs, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete txs from the db: ", result).c_str())); + + RENAME_DB("txr"); + + mdb_dbi_close(m_env, m_txs); + + lmdb_db_open(txn, "txs", MDB_INTEGERKEY, m_txs, "Failed to open db handle for txs"); + + txn.commit(); + } while(0); + + uint32_t version = 1; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_copy<const char *> vk("version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + +void BlockchainLMDB::migrate(const uint32_t oldversion) +{ + switch(oldversion) { + case 0: + migrate_0_1(); /* FALLTHRU */ + default: + ; + } +} + } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 6cd3e0e8f..c7121bf63 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -43,20 +43,13 @@ typedef struct mdb_txn_cursors { MDB_cursor *m_txc_blocks; MDB_cursor *m_txc_block_heights; - MDB_cursor *m_txc_block_hashes; - MDB_cursor *m_txc_block_timestamps; - MDB_cursor *m_txc_block_sizes; - MDB_cursor *m_txc_block_diffs; - MDB_cursor *m_txc_block_coins; + MDB_cursor *m_txc_block_info; MDB_cursor *m_txc_output_txs; - MDB_cursor *m_txc_output_indices; MDB_cursor *m_txc_output_amounts; - MDB_cursor *m_txc_output_keys; MDB_cursor *m_txc_txs; - MDB_cursor *m_txc_tx_heights; - MDB_cursor *m_txc_tx_unlocks; + MDB_cursor *m_txc_tx_indices; MDB_cursor *m_txc_tx_outputs; MDB_cursor *m_txc_spent_keys; @@ -66,18 +59,11 @@ typedef struct mdb_txn_cursors #define m_cur_blocks m_cursors->m_txc_blocks #define m_cur_block_heights m_cursors->m_txc_block_heights -#define m_cur_block_hashes m_cursors->m_txc_block_hashes -#define m_cur_block_timestamps m_cursors->m_txc_block_timestamps -#define m_cur_block_sizes m_cursors->m_txc_block_sizes -#define m_cur_block_diffs m_cursors->m_txc_block_diffs -#define m_cur_block_coins m_cursors->m_txc_block_coins +#define m_cur_block_info m_cursors->m_txc_block_info #define m_cur_output_txs m_cursors->m_txc_output_txs -#define m_cur_output_indices m_cursors->m_txc_output_indices #define m_cur_output_amounts m_cursors->m_txc_output_amounts -#define m_cur_output_keys m_cursors->m_txc_output_keys #define m_cur_txs m_cursors->m_txc_txs -#define m_cur_tx_heights m_cursors->m_txc_tx_heights -#define m_cur_tx_unlocks m_cursors->m_txc_tx_unlocks +#define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_hf_versions m_cursors->m_txc_hf_versions @@ -87,18 +73,11 @@ typedef struct mdb_rflags bool m_rf_txn; bool m_rf_blocks; bool m_rf_block_heights; - bool m_rf_block_hashes; - bool m_rf_block_timestamps; - bool m_rf_block_sizes; - bool m_rf_block_diffs; - bool m_rf_block_coins; + bool m_rf_block_info; bool m_rf_output_txs; - bool m_rf_output_indices; bool m_rf_output_amounts; - bool m_rf_output_keys; bool m_rf_txs; - bool m_rf_tx_heights; - bool m_rf_tx_unlocks; + bool m_rf_tx_indices; bool m_rf_tx_outputs; bool m_rf_spent_keys; bool m_rf_hf_versions; @@ -223,6 +202,7 @@ public: virtual uint64_t height() const; virtual bool tx_exists(const crypto::hash& h) const; + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const; virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; @@ -246,10 +226,8 @@ public: virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices); - virtual void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &indices); - virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const; - virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const; + virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const; virtual bool has_key_image(const crypto::key_image& img) const; @@ -306,18 +284,23 @@ private: virtual void remove_block(); - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time); + virtual uint64_t add_output(const crypto::hash& tx_hash, + const tx_out& tx_output, + const uint64_t& local_index, + const uint64_t unlock_time + ); - virtual void remove_output(const tx_out& tx_output); + virtual void add_tx_amount_output_indices(const uint64_t tx_id, + const std::vector<uint64_t>& amount_output_indices + ); - void remove_tx_outputs(const MDB_val *tx_hash, const transaction& tx); + void remove_tx_outputs(const uint64_t tx_id, const transaction& tx); - void remove_output(const MDB_val *out_index, const uint64_t amount); - void remove_amount_output_index(const uint64_t amount, const MDB_val *global_output_index); + void remove_output(const uint64_t amount, const uint64_t& out_index); virtual void add_spent_key(const crypto::key_image& k_image); @@ -349,16 +332,6 @@ private: */ tx_out output_from_blob(const blobdata& blob) const; - /** - * @brief get the global index of the index-th output of the given amount - * - * @param amount the output amount - * @param index the index into the set of outputs of that amount - * - * @return the global index of the desired output - */ - uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); - void check_open() const; virtual bool is_read_only() const; @@ -366,25 +339,24 @@ private: // fix up anything that may be wrong due to past bugs virtual void fixup(); + // migrate from older DB version to current + void migrate(const uint32_t oldversion); + + // migrate from DB version 0 to 1 + void migrate_0_1(); + MDB_env* m_env; MDB_dbi m_blocks; MDB_dbi m_block_heights; - MDB_dbi m_block_hashes; - MDB_dbi m_block_timestamps; - MDB_dbi m_block_sizes; - MDB_dbi m_block_diffs; - MDB_dbi m_block_coins; + MDB_dbi m_block_info; MDB_dbi m_txs; - MDB_dbi m_tx_unlocks; - MDB_dbi m_tx_heights; + MDB_dbi m_tx_indices; MDB_dbi m_tx_outputs; MDB_dbi m_output_txs; - MDB_dbi m_output_indices; MDB_dbi m_output_amounts; - MDB_dbi m_output_keys; MDB_dbi m_spent_keys; @@ -394,6 +366,7 @@ private: MDB_dbi m_properties; uint64_t m_height; + uint64_t m_num_txs; uint64_t m_num_outputs; mutable uint64_t m_cum_size; // used in batch size estimation mutable int m_cum_count; diff --git a/src/blockchain_utilities/cn_deserialize.cpp b/src/blockchain_utilities/cn_deserialize.cpp index bf02dc150..a8448dcee 100644 --- a/src/blockchain_utilities/cn_deserialize.cpp +++ b/src/blockchain_utilities/cn_deserialize.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/tx_extra.h" #include "cryptonote_core/blockchain.h" #include "blockchain_utilities.h" #include "common/command_line.h" @@ -140,6 +141,7 @@ int main(int argc, char* argv[]) cryptonote::block block; cryptonote::transaction tx; + std::vector<cryptonote::tx_extra_field> fields; if (cryptonote::parse_and_validate_block_from_blob(blob, block)) { std::cout << "Parsed block:" << std::endl; @@ -149,6 +151,26 @@ int main(int argc, char* argv[]) { std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; + + if (cryptonote::parse_tx_extra(tx.extra, fields)) + { + std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl; + for (size_t n = 0; n < fields.size(); ++n) + { + std::cout << "field " << n << ": "; + if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get<cryptonote::tx_extra_padding>(fields[n]).size << " bytes"; + else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get<cryptonote::tx_extra_pub_key>(fields[n]).pub_key; + else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_nonce>(fields[n]).nonce); + else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).depth << ", merkle root " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).merkle_root; + else if (typeid(cryptonote::tx_extra_mysterious_minergate) == fields[n].type()) std::cout << "extra minergate custom: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_mysterious_minergate>(fields[n]).data); + else std::cout << "unknown"; + std::cout << std::endl; + } + } + else + { + std::cout << "Failed to parse tx_extra" << std::endl; + } } else { diff --git a/src/common/json_util.h b/src/common/json_util.h new file mode 100644 index 000000000..6f8b4c18f --- /dev/null +++ b/src/common/json_util.h @@ -0,0 +1,53 @@ +// Copyright (c) 2016, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#define GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, name, type, jtype, mandatory) \ + type field_##name; \ + bool field_##name##_found = false; \ + (void)field_##name##_found; \ + do if (json.HasMember(#name)) \ + { \ + if (json[#name].Is##jtype()) \ + { \ + field_##name = json[#name].Get##jtype(); \ + field_##name##_found = true; \ + } \ + else \ + { \ + LOG_ERROR("Field " << #name << " found in JSON, but not " << #jtype); \ + return false; \ + } \ + } \ + else if (mandatory) \ + { \ + LOG_ERROR("Field " << #name << " not found in JSON"); \ + return false; \ + } while(0) + diff --git a/src/common/util.cpp b/src/common/util.cpp index 6f75e5bad..a53a9be52 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -413,4 +413,36 @@ std::string get_nix_version_display_string() } return false; } + void set_strict_default_file_permissions(bool strict) + { +#if defined(__MINGW32__) || defined(__MINGW__) + // no clue about the odd one out +#else + mode_t mode = strict ? 077 : 0; + umask(mode); +#endif + } + + namespace + { + boost::mutex max_concurrency_lock; + unsigned max_concurrency = boost::thread::hardware_concurrency(); + } + + void set_max_concurrency(unsigned n) + { + if (n < 1) + n = boost::thread::hardware_concurrency(); + unsigned hwc = boost::thread::hardware_concurrency(); + if (n > hwc) + n = hwc; + boost::lock_guard<boost::mutex> lock(max_concurrency_lock); + max_concurrency = n; + } + + unsigned get_max_concurrency() + { + boost::lock_guard<boost::mutex> lock(max_concurrency_lock); + return max_concurrency; + } } diff --git a/src/common/util.h b/src/common/util.h index 7554b1df7..4fcf66b8f 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -158,4 +158,9 @@ namespace tools /*! \brief where the installed handler is stored */ static std::function<void(int)> m_handler; }; + + void set_strict_default_file_permissions(bool strict); + + void set_max_concurrency(unsigned n); + unsigned get_max_concurrency(); } diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index e47aab0f7..f5e655274 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -83,7 +83,7 @@ namespace crypto { /* generate a random 32-byte (256-bit) integer and copy it to res */ static inline void random_scalar(ec_scalar &res) { unsigned char tmp[64]; - generate_random_bytes(64, tmp); + generate_random_bytes_not_thread_safe(64, tmp); sc_reduce(tmp); memcpy(&res, tmp, 32); } @@ -263,16 +263,17 @@ namespace crypto { PUSH_WARNINGS DISABLE_VS_WARNINGS(4200) + struct ec_point_pair { + ec_point a, b; + }; struct rs_comm { hash h; - struct { - ec_point a, b; - } ab[]; + struct ec_point_pair ab[]; }; POP_WARNINGS static inline size_t rs_comm_size(size_t pubs_count) { - return sizeof(rs_comm) + pubs_count * sizeof(rs_comm().ab[0]); + return sizeof(rs_comm) + pubs_count * sizeof(ec_point_pair); } void crypto_ops::generate_ring_signature(const hash &prefix_hash, const key_image &image, diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 883aa521a..fa55c2aab 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -117,13 +117,20 @@ namespace crypto { const public_key *const *, std::size_t, const signature *); }; + /* Generate N random bytes + */ + inline void rand(size_t N, uint8_t *bytes) { + boost::lock_guard<boost::mutex> lock(random_lock); + generate_random_bytes_not_thread_safe(N, bytes); + } + /* Generate a value filled with random bytes. */ template<typename T> typename std::enable_if<std::is_pod<T>::value, T>::type rand() { typename std::remove_cv<T>::type res; boost::lock_guard<boost::mutex> lock(random_lock); - generate_random_bytes(sizeof(T), &res); + generate_random_bytes_not_thread_safe(sizeof(T), &res); return res; } diff --git a/src/crypto/oaes_lib.c b/src/crypto/oaes_lib.c index 7fbe96e4c..4c1446898 100644 --- a/src/crypto/oaes_lib.c +++ b/src/crypto/oaes_lib.c @@ -27,10 +27,6 @@ * POSSIBILITY OF SUCH DAMAGE. * --------------------------------------------------------------------------- */ -static const char _NR[] = { - 0x4e,0x61,0x62,0x69,0x6c,0x20,0x53,0x2e,0x20, - 0x41,0x6c,0x20,0x52,0x61,0x6d,0x6c,0x69,0x00 }; - #include <stddef.h> #include <time.h> #include <string.h> diff --git a/src/crypto/random.c b/src/crypto/random.c index d7fcb7e65..6a9f63c12 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -45,7 +45,7 @@ static void generate_system_random_bytes(size_t n, void *result); static void generate_system_random_bytes(size_t n, void *result) { HCRYPTPROV prov; -#define must_succeed(x) do if (!(x)) assert(0); while (0) +#define must_succeed(x) do if (!(x)) abort(); while (0) must_succeed(CryptAcquireContext(&prov, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT | CRYPT_SILENT)); must_succeed(CryptGenRandom(prov, (DWORD)n, result)); must_succeed(CryptReleaseContext(prov, 0)); @@ -113,7 +113,7 @@ INITIALIZER(init_random) { #endif } -void generate_random_bytes(size_t n, void *result) { +void generate_random_bytes_not_thread_safe(size_t n, void *result) { #if !defined(NDEBUG) assert(curstate == 1); curstate = 2; diff --git a/src/crypto/random.h b/src/crypto/random.h index 322b5bad1..b0d2303b6 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -32,4 +32,4 @@ #include <stddef.h> -void generate_random_bytes(size_t n, void *result); +void generate_random_bytes_not_thread_safe(size_t n, void *result); diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 4efa8af6c..8a1807657 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -658,10 +658,6 @@ void slow_hash_free_state(void) return; } -static void (*const extra_hashes[4])(const void *, size_t, char *) = { - hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein -}; - #define MEMORY (1 << 21) /* 2 MiB */ #define ITER (1 << 20) #define AES_BLOCK_SIZE 16 @@ -683,7 +679,8 @@ static void (*const extra_hashes[4])(const void *, size_t, char *) = { #include "aesb.c" -/* The asm corresponds to this C code +#ifndef ARM_MUL_IMPL_ASM +/* The asm corresponds to this C code */ #define SHORT uint32_t #define LONG uint64_t @@ -714,7 +711,8 @@ void mul(const uint8_t *ca, const uint8_t *cb, uint8_t *cres) { res[3] = t.tmp[2]; res[0] = t.tmp[6]; res[1] = t.tmp[7]; -} */ +} +#else // ARM_MUL_IMPL_ASM (TODO: this fails hash-slow test with GCC 6.1.1) /* Can work as inline, but actually runs slower. Keep it separate */ #define mul(a, b, c) cn_mul128(a, b, c) @@ -749,6 +747,7 @@ __asm__ __volatile__( : [A]"r"(aa[1]), [a]"r"(aa[0]), [B]"r"(bb[1]), [b]"r"(bb[0]), [r]"r"(r) : "cc", "memory"); } +#endif // ARM_MUL_IMPL_ASM STATIC INLINE void sum_half_blocks(uint8_t* a, const uint8_t* b) { @@ -824,7 +823,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash) { for(j = 0; j < INIT_SIZE_BLK; j++) aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], expandedKey); - memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } U64(a)[0] = U64(&state.k[0])[0] ^ U64(&state.k[32])[0]; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index da14d7575..d332f8997 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -498,6 +498,7 @@ block Blockchain::pop_block_from_blockchain() } } } + update_next_cumulative_size_limit(); m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); return popped_block; @@ -1970,14 +1971,15 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - if (!m_db->tx_exists(tx_id)) + uint64_t tx_index; + if (!m_db->tx_exists(tx_id, tx_index)) { LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); return false; } // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts - indexs = m_db->get_tx_amount_output_indices(tx_id); + indexs = m_db->get_tx_amount_output_indices(tx_index); CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty"); return true; @@ -1991,7 +1993,7 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u // This function overloads its sister function with // an extra value (hash of highest block that holds an output used as input) // as a return-by-reference. -bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id, bool kept_by_block) +bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block) { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2013,26 +2015,20 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block #endif TIME_MEASURE_START(a); - bool res = check_tx_inputs(tx, &max_used_block_height); + bool res = check_tx_inputs(tx, tvc, &max_used_block_height); TIME_MEASURE_FINISH(a); - crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); if(m_show_time_stats) LOG_PRINT_L0("HASH: " << "+" << " VIN/VOUT: " << tx.vin.size() << "/" << tx.vout.size() << " H: " << max_used_block_height << " chcktx: " << a + m_fake_scan_time); if (!res) return false; - // ND: Speedup: - // 1. keep a list of verified transactions, when the Blockchain tries to check a tx again, - // verify against list and skip if already verified to be correct. - m_check_tx_inputs_table.emplace(tx_prefix_hash, std::make_pair(res, max_used_block_height)); - CHECK_AND_ASSERT_MES(max_used_block_height < m_db->height(), false, "internal error: max used block index=" << max_used_block_height << " is not less then blockchain size = " << m_db->height()); max_used_block_id = m_db->get_block_hash_from_height(max_used_block_height); return true; } //------------------------------------------------------------------ -bool Blockchain::check_tx_outputs(const transaction& tx) +bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context &tvc) { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2041,6 +2037,7 @@ bool Blockchain::check_tx_outputs(const transaction& tx) if (m_hardfork->get_current_version() >= 2) { for (auto &o: tx.vout) { if (!is_valid_decomposed_amount(o.amount)) { + tvc.m_invalid_output = true; return false; } } @@ -2066,7 +2063,7 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const // check_tx_input() rather than here, and use this function simply // to iterate the inputs as necessary (splitting the task // using threads, etc.) -bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) +bool Blockchain::check_tx_inputs(const transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height) { LOG_PRINT_L3("Blockchain::" << __func__); size_t sig_index = 0; @@ -2075,16 +2072,6 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); - auto its = m_check_tx_inputs_table.find(tx_prefix_hash); - if (its != m_check_tx_inputs_table.end()) - { - if (!its->second.first) - return false; - if (pmax_used_block_height) - *pmax_used_block_height = its->second.second; - return true; - } - // from hard fork 2, we require mixin at least 2 unless one output cannot mix with 2 others // if one output cannot mix with 2 others, we accept at most 1 output that can mix if (m_hardfork->get_current_version() >= 2) @@ -2113,11 +2100,13 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc if (n_unmixable == 0) { LOG_PRINT_L1("Tx " << get_transaction_hash(tx) << " has too low mixin (" << mixin << "), and no unmixable inputs"); + tvc.m_low_mixin = true; return false; } if (n_mixable > 1) { LOG_PRINT_L1("Tx " << get_transaction_hash(tx) << " has too low mixin (" << mixin << "), and more than one mixable input with unmixable inputs"); + tvc.m_low_mixin = true; return false; } } @@ -2136,7 +2125,7 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc std::vector < uint64_t > results; results.resize(tx.vin.size(), 0); - int threads = boost::thread::hardware_concurrency(); + int threads = tools::get_max_concurrency(); boost::asio::io_service ioservice; boost::thread_group threadpool; @@ -2156,6 +2145,7 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc if(ioservice_active) \ { \ work.reset(); \ + while (!ioservice.stopped()) ioservice.poll(); \ threadpool.join_all(); \ ioservice.stop(); \ ioservice_active = false; \ @@ -2176,6 +2166,7 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc if(have_tx_keyimg_as_spent(in_to_key.k_image)) { LOG_PRINT_L1("Key image already spent in blockchain: " << epee::string_tools::pod_to_hex(in_to_key.k_image)); + tvc.m_double_spend = true; return false; } @@ -2667,7 +2658,8 @@ leave: #endif { // validate that transaction inputs and the keys spending them are correct. - if(!check_tx_inputs(tx)) + tx_verification_context tvc; + if(!check_tx_inputs(tx, tvc)) { LOG_PRINT_L1("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs."); @@ -2722,7 +2714,11 @@ leave: // populate various metadata about the block to be stored alongside it. block_size = cumulative_block_size; cumulative_difficulty = current_diffic; - already_generated_coins = already_generated_coins + base_reward; + // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of + // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins + // at MONEY_SUPPLY. already_generated_coins is only used to compute the block subsidy and MONEY_SUPPLY yields a + // subsidy of 0 under the base formula and therefore the minimum subsidy >0 in the tail state. + already_generated_coins = base_reward < (MONEY_SUPPLY-already_generated_coins) ? already_generated_coins + base_reward : MONEY_SUPPLY; if(m_db->height()) cumulative_difficulty += m_db->get_block_cumulative_difficulty(m_db->height() - 1); @@ -2962,7 +2958,6 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) TIME_MEASURE_FINISH(t1); m_blocks_longhash_table.clear(); m_scan_table.clear(); - m_check_tx_inputs_table.clear(); m_blocks_txs_check.clear(); m_check_txin_table.clear(); @@ -3007,7 +3002,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e return true; bool blocks_exist = false; - uint64_t threads = boost::thread::hardware_concurrency(); + uint64_t threads = tools::get_max_concurrency(); if (blocks_entry.size() > 1 && threads > 1 && m_max_prepare_blocks_threads > 1) { @@ -3110,7 +3105,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e m_fake_pow_calc_time = 0; m_scan_table.clear(); - m_check_tx_inputs_table.clear(); m_check_txin_table.clear(); TIME_MEASURE_FINISH(prepare); @@ -3207,7 +3201,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e // [output] stores all transactions for each tx_out_index::hash found std::vector<std::unordered_map<crypto::hash, cryptonote::transaction>> transactions(amounts.size()); - threads = boost::thread::hardware_concurrency(); + threads = tools::get_max_concurrency(); if (!m_db->can_thread_bulk_indices()) threads = 1; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 6bae0364d..21086d578 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -472,11 +472,12 @@ namespace cryptonote * @param tx the transaction to validate * @param pmax_used_block_height return-by-reference block height of most recent input * @param max_used_block_id return-by-reference block hash of most recent input + * @param tvc returned information about tx verification * @param kept_by_block whether or not the transaction is from a previously-verified block * * @return false if any input is invalid, otherwise true */ - bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, bool kept_by_block = false); + bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false); /** * @brief check that a transaction's outputs conform to current standards @@ -486,10 +487,11 @@ namespace cryptonote * written out would have only one non-zero digit in base 10). * * @param tx the transaction to check the outputs of + * @param tvc returned info about tx verification * * @return false if any outputs do not conform, otherwise true */ - bool check_tx_outputs(const transaction& tx); + bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc); /** * @brief gets the blocksize limit based on recent blocks @@ -788,7 +790,6 @@ namespace cryptonote // metadata containers std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>> m_scan_table; - std::unordered_map<crypto::hash, std::pair<bool, uint64_t>> m_check_tx_inputs_table; std::unordered_map<crypto::hash, crypto::hash> m_blocks_longhash_table; std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, bool>> m_check_txin_table; @@ -883,11 +884,12 @@ namespace cryptonote * transaction. * * @param tx the transaction to validate + * @param tvc returned information about tx verification * @param pmax_related_block_height return-by-pointer the height of the most recent block in the input set * * @return false if any validation step fails, otherwise true */ - bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL); + bool check_tx_inputs(const transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height = NULL); /** * @brief performs a blockchain reorganization according to the longest chain rule diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index c31be5acf..20b9f0b0b 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -489,6 +489,7 @@ namespace cryptonote { LOG_PRINT_L1("WRONG TRANSACTION BLOB, too big size " << tx_blob.size() << ", rejected"); tvc.m_verifivation_failed = true; + tvc.m_too_big = true; return false; } diff --git a/src/cryptonote_core/cryptonote_format_utils.cpp b/src/cryptonote_core/cryptonote_format_utils.cpp index 3c1acd8a9..ff752ae47 100644 --- a/src/cryptonote_core/cryptonote_format_utils.cpp +++ b/src/cryptonote_core/cryptonote_format_utils.cpp @@ -145,10 +145,18 @@ namespace cryptonote [&out_amounts](uint64_t a_dust) { out_amounts.push_back(a_dust); }); CHECK_AND_ASSERT_MES(1 <= max_outs, false, "max_out must be non-zero"); - while (max_outs < out_amounts.size()) + if (height == 0) { - out_amounts[out_amounts.size() - 2] += out_amounts.back(); - out_amounts.resize(out_amounts.size() - 1); + // the genesis block was not decomposed, for unknown reasons + while (max_outs < out_amounts.size()) + { + out_amounts[out_amounts.size() - 2] += out_amounts.back(); + out_amounts.resize(out_amounts.size() - 1); + } + } + else + { + CHECK_AND_ASSERT_MES(max_outs >= out_amounts.size(), false, "max_out exceeded"); } uint64_t summary_amounts = 0; @@ -283,14 +291,14 @@ namespace cryptonote { tx_extra_field field; bool r = ::do_serialize(ar, field); - CHECK_AND_NO_ASSERT_MES(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); tx_extra_fields.push_back(field); std::ios_base::iostate state = iss.rdstate(); eof = (EOF == iss.peek()); iss.clear(state); } - CHECK_AND_NO_ASSERT_MES(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); + CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); return true; } @@ -349,7 +357,7 @@ namespace cryptonote { tx_extra_field field; bool r = ::do_serialize(ar, field); - CHECK_AND_NO_ASSERT_MES(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); if (field.type() != typeid(tx_extra_nonce)) ::do_serialize(newar, field); @@ -357,7 +365,7 @@ namespace cryptonote eof = (EOF == iss.peek()); iss.clear(state); } - CHECK_AND_NO_ASSERT_MES(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); + CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); tx_extra.clear(); std::string s = oss.str(); tx_extra.reserve(s.size()); diff --git a/src/cryptonote_core/cryptonote_format_utils.h b/src/cryptonote_core/cryptonote_format_utils.h index 4142829e2..056940e98 100644 --- a/src/cryptonote_core/cryptonote_format_utils.h +++ b/src/cryptonote_core/cryptonote_format_utils.h @@ -44,7 +44,7 @@ namespace cryptonote crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx); - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 1, uint8_t hard_fork_version = 1); + bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key); bool decrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key); diff --git a/src/cryptonote_core/tx_extra.h b/src/cryptonote_core/tx_extra.h index 93d2e21b3..6f5fbe466 100644 --- a/src/cryptonote_core/tx_extra.h +++ b/src/cryptonote_core/tx_extra.h @@ -38,6 +38,7 @@ #define TX_EXTRA_TAG_PUBKEY 0x01 #define TX_EXTRA_NONCE 0x02 #define TX_EXTRA_MERGE_MINING_TAG 0x03 +#define TX_EXTRA_MYSTERIOUS_MINERGATE_TAG 0xDE #define TX_EXTRA_NONCE_PAYMENT_ID 0x00 #define TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID 0x01 @@ -158,14 +159,24 @@ namespace cryptonote } }; + struct tx_extra_mysterious_minergate + { + std::string data; + + BEGIN_SERIALIZE() + FIELD(data) + END_SERIALIZE() + }; + // tx_extra_field format, except tx_extra_padding and tx_extra_pub_key: // varint tag; // varint size; // varint data[]; - typedef boost::variant<tx_extra_padding, tx_extra_pub_key, tx_extra_nonce, tx_extra_merge_mining_tag> tx_extra_field; + typedef boost::variant<tx_extra_padding, tx_extra_pub_key, tx_extra_nonce, tx_extra_merge_mining_tag, tx_extra_mysterious_minergate> tx_extra_field; } VARIANT_TAG(binary_archive, cryptonote::tx_extra_padding, TX_EXTRA_TAG_PADDING); VARIANT_TAG(binary_archive, cryptonote::tx_extra_pub_key, TX_EXTRA_TAG_PUBKEY); VARIANT_TAG(binary_archive, cryptonote::tx_extra_nonce, TX_EXTRA_NONCE); VARIANT_TAG(binary_archive, cryptonote::tx_extra_merge_mining_tag, TX_EXTRA_MERGE_MINING_TAG); +VARIANT_TAG(binary_archive, cryptonote::tx_extra_mysterious_minergate, TX_EXTRA_MYSTERIOUS_MINERGATE_TAG); diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 5d67acdd2..3d5ab86e1 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -54,6 +54,11 @@ namespace cryptonote { namespace { + //TODO: constants such as these should at least be in the header, + // but probably somewhere more accessible to the rest of the + // codebase. As it stands, it is at best nontrivial to test + // whether or not changing these parameters (or adding new) + // will work correctly. size_t const TRANSACTION_SIZE_LIMIT_V1 = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); size_t const TRANSACTION_SIZE_LIMIT_V2 = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); time_t const MIN_RELAY_TIME = (60 * 5); // only start re-relaying transactions after that many seconds @@ -97,6 +102,7 @@ namespace cryptonote if(!check_inputs_types_supported(tx)) { tvc.m_verifivation_failed = true; + tvc.m_invalid_input = true; return false; } @@ -113,17 +119,20 @@ namespace cryptonote { LOG_PRINT_L1("transaction use more money then it has: use " << print_money(outputs_amount) << ", have " << print_money(inputs_amount)); tvc.m_verifivation_failed = true; + tvc.m_overspend = true; return false; } + // fee per kilobyte, size rounded up. uint64_t fee = inputs_amount - outputs_amount; uint64_t needed_fee = blob_size / 1024; needed_fee += (blob_size % 1024) ? 1 : 0; needed_fee *= FEE_PER_KB; - if (!kept_by_block && fee < needed_fee /*&& fee < MINING_ALLOWED_LEGACY_FEE*/) + if (!kept_by_block && fee < needed_fee) { LOG_PRINT_L1("transaction fee is not enough: " << print_money(fee) << ", minimum fee: " << print_money(needed_fee)); tvc.m_verifivation_failed = true; + tvc.m_fee_too_low = true; return false; } @@ -132,40 +141,46 @@ namespace cryptonote { LOG_PRINT_L1("transaction is too big: " << blob_size << " bytes, maximum size: " << tx_size_limit); tvc.m_verifivation_failed = true; + tvc.m_too_big = true; return false; } - //check key images for transaction if it is not kept by block + // if the transaction came from a block popped from the chain, + // don't check if we have its key images as spent. + // TODO: Investigate why not? if(!kept_by_block) { if(have_tx_keyimges_as_spent(tx)) { LOG_PRINT_L1("Transaction with id= "<< id << " used already spent key images"); tvc.m_verifivation_failed = true; + tvc.m_double_spend = true; return false; } } - if (!m_blockchain.check_tx_outputs(tx)) + if (!m_blockchain.check_tx_outputs(tx, tvc)) { LOG_PRINT_L1("Transaction with id= "<< id << " has at least one invalid outout"); tvc.m_verifivation_failed = true; + tvc.m_invalid_output = true; return false; } crypto::hash max_used_block_id = null_hash; uint64_t max_used_block_height = 0; #if BLOCKCHAIN_DB == DB_LMDB - bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id, kept_by_block); + bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id, tvc, kept_by_block); #else bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id); #endif CRITICAL_REGION_LOCAL(m_transactions_lock); if(!ch_inp_res) { + // if the transaction was valid before (kept_by_block), then it + // may become valid again, so ignore the failed inputs check. if(kept_by_block) { - //anyway add this transaction to pool, because it related to block auto txd_p = m_transactions.insert(transactions_container::value_type(id, tx_details())); CHECK_AND_ASSERT_MES(txd_p.second, false, "transaction already exists at inserting in memory pool"); txd_p.first->second.blob_size = blob_size; @@ -207,13 +222,14 @@ namespace cryptonote tvc.m_should_be_relayed = true; } + // assume failure during verification steps until success is certain tvc.m_verifivation_failed = true; - //update image_keys container, here should everything goes ok. + BOOST_FOREACH(const auto& in, tx.vin) { CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, txin, false); std::unordered_set<crypto::hash>& kei_image_set = m_spent_key_images[txin.k_image]; - CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: keeped_by_block=" << kept_by_block + CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: kept_by_block=" << kept_by_block << ", kei_image_set.size()=" << kei_image_set.size() << ENDL << "txin.k_image=" << txin.k_image << ENDL << "tx_id=" << id ); auto ins_res = kei_image_set.insert(id); @@ -223,7 +239,7 @@ namespace cryptonote tvc.m_verifivation_failed = false; m_txs_by_fee.emplace((double)blob_size / fee, id); - //succeed + return true; } //--------------------------------------------------------------------------------- @@ -235,6 +251,9 @@ namespace cryptonote return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, version); } //--------------------------------------------------------------------------------- + //FIXME: Can return early before removal of all of the key images. + // At the least, need to make sure that a false return here + // is treated properly. Should probably not return early, however. bool tx_memory_pool::remove_transaction_keyimages(const transaction& tx) { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -301,7 +320,7 @@ namespace cryptonote ); } //--------------------------------------------------------------------------------- - //proper tx_pool handling courtesy of CryptoZoidberg and Boolberry + //TODO: investigate whether boolean return is appropriate bool tx_memory_pool::remove_stuck_transactions() { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -332,6 +351,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- + //TODO: investigate whether boolean return is appropriate bool tx_memory_pool::get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::transaction>> &txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -380,6 +400,7 @@ namespace cryptonote txs.push_back(tx_vt.second.tx); } //------------------------------------------------------------------ + //TODO: investigate whether boolean return is appropriate bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -480,7 +501,8 @@ namespace cryptonote if(txd.last_failed_id != null_hash && m_blockchain.get_current_blockchain_height() > txd.last_failed_height && txd.last_failed_id == m_blockchain.get_block_id_by_height(txd.last_failed_height)) return false;//we already sure that this tx is broken for this height - if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id)) + tx_verification_context tvc; + if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -490,13 +512,18 @@ namespace cryptonote { if(txd.max_used_block_height >= m_blockchain.get_current_blockchain_height()) return false; - if(m_blockchain.get_block_id_by_height(txd.max_used_block_height) != txd.max_used_block_id) + if(true) { //if we already failed on this height and id, skip actual ring signature check if(txd.last_failed_id == m_blockchain.get_block_id_by_height(txd.last_failed_height)) return false; //check ring signature again, it is possible (with very small chance) that this transaction become again valid +#if BLOCKCHAIN_DB == DB_LMDB + tx_verification_context tvc; + if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) +#else if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id)) +#endif { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -556,6 +583,7 @@ namespace cryptonote return ss.str(); } //--------------------------------------------------------------------------------- + //TODO: investigate whether boolean return is appropriate bool tx_memory_pool::fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee) { // Warning: This function takes already_generated_ @@ -646,6 +674,7 @@ namespace cryptonote return n_removed; } //--------------------------------------------------------------------------------- + //TODO: investigate whether only ever returning true is correct bool tx_memory_pool::init(const std::string& config_folder) { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -679,6 +708,7 @@ namespace cryptonote } //--------------------------------------------------------------------------------- + //TODO: investigate whether only ever returning true is correct bool tx_memory_pool::deinit() { if (m_config_folder.empty()) diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 71febcab6..c7aab7f08 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -57,7 +57,9 @@ namespace cryptonote /* */ /************************************************************************/ + //! pair of <transaction fee, transaction hash> for organization typedef std::pair<double, crypto::hash> tx_by_fee_entry; + class txCompare { public: @@ -71,47 +73,256 @@ namespace cryptonote } }; + //! container for sorting transactions by fee per unit size typedef std::set<tx_by_fee_entry, txCompare> sorted_tx_container; + /** + * @brief Transaction pool, handles transactions which are not part of a block + * + * This class handles all transactions which have been received, but not as + * part of a block. + * + * This handling includes: + * storing the transactions + * organizing the transactions by fee per size + * taking/giving transactions to and from various other components + * saving the transactions to disk on shutdown + * helping create a new block template by choosing transactions for it + * + */ class tx_memory_pool: boost::noncopyable { public: #if BLOCKCHAIN_DB == DB_LMDB + /** + * @brief Constructor + * + * @param bchs a Blockchain class instance, for getting chain info + */ tx_memory_pool(Blockchain& bchs); #else tx_memory_pool(blockchain_storage& bchs); #endif - bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, uint8_t version); - bool add_tx(const transaction &tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, uint8_t version); - //gets tx and remove it from pool + + + /** + * @copydoc add_tx(const transaction&, tx_verification_context&, bool, bool, uint8_t) + * + * @param id the transaction's hash + * @param blob_size the transaction's size + */ + bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, uint8_t version); + + /** + * @brief add a transaction to the transaction pool + * + * Most likely the transaction will come from the network, but it is + * also possible for transactions to come from popped blocks during + * a reorg, or from local clients creating a transaction and + * submitting it to the network + * + * @param tx the transaction to be added + * @param tvc return-by-reference status about the transaction verification + * @param kept_by_block has this transaction been in a block? + * @param relayed was this transaction from the network or a local client? + * @param version the version used to create the transaction + * + * @return true if the transaction passes validations, otherwise false + */ + bool add_tx(const transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool relayed, uint8_t version); + + /** + * @brief takes a transaction with the given hash from the pool + * + * @param id the hash of the transaction + * @param tx return-by-reference the transaction taken + * @param blob_size return-by-reference the transaction's size + * @param fee the transaction fee + * @param relayed return-by-reference was transaction relayed to us by the network? + * + * @return true unless the transaction cannot be found in the pool + */ bool take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed); + /** + * @brief checks if the pool has a transaction with the given hash + * + * @param id the hash to look for + * + * @return true if the transaction is in the pool, otherwise false + */ bool have_tx(const crypto::hash &id) const; + + /** + * @brief action to take when notified of a block added to the blockchain + * + * Currently does nothing + * + * @param new_block_height the height of the blockchain after the change + * @param top_block_id the hash of the new top block + * + * @return true + */ bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id); + + /** + * @brief action to take when notified of a block removed from the blockchain + * + * Currently does nothing + * + * @param new_block_height the height of the blockchain after the change + * @param top_block_id the hash of the new top block + * + * @return true + */ bool on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id); + + /** + * @brief action to take periodically + * + * Currently checks transaction pool for stale ("stuck") transactions + */ void on_idle(); + /** + * @brief locks the transaction pool + */ void lock() const; + + /** + * @brief unlocks the transaction pool + */ void unlock() const; // load/store operations + + /** + * @brief loads pool state (if any) from disk, and initializes pool + * + * @param config_folder folder name where pool state will be + * + * @return true + */ bool init(const std::string& config_folder); + + /** + * @brief attempts to save the transaction pool state to disk + * + * Currently fails (returns false) if the data directory from init() + * does not exist and cannot be created, but returns true even if + * saving to disk is unsuccessful. + * + * @return true in most cases (see above) + */ bool deinit(); + + /** + * @brief Chooses transactions for a block to include + * + * @param bl return-by-reference the block to fill in with transactions + * @param median_size the current median block size + * @param already_generated_coins the current total number of coins "minted" + * @param total_size return-by-reference the total size of the new block + * @param fee return-by-reference the total of fees from the included transactions + * + * @return true + */ bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee); + + /** + * @brief get a list of all transactions in the pool + * + * @param txs return-by-reference the list of transactions + */ void get_transactions(std::list<transaction>& txs) const; + + /** + * @brief get information about all transactions and key images in the pool + * + * see documentation on tx_info and spent_key_image_info for more details + * + * @param tx_infos return-by-reference the transactions' information + * @param key_image_infos return-by-reference the spent key images' information + * + * @return true + */ bool get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const; + + /** + * @brief get a specific transaction from the pool + * + * @param h the hash of the transaction to get + * @param tx return-by-reference the transaction requested + * + * @return true if the transaction is found, otherwise false + */ bool get_transaction(const crypto::hash& h, transaction& tx) const; + + /** + * @brief get a list of all relayable transactions and their hashes + * + * "relayable" in this case means: + * nonzero fee + * hasn't been relayed too recently + * isn't old enough that relaying it is considered harmful + * + * @param txs return-by-reference the transactions and their hashes + * + * @return true + */ bool get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::transaction>>& txs) const; + + /** + * @brief tell the pool that certain transactions were just relayed + * + * @param txs the list of transactions (and their hashes) + */ void set_relayed(const std::list<std::pair<crypto::hash, cryptonote::transaction>>& txs); + + /** + * @brief get the total number of transactions in the pool + * + * @return the number of transactions in the pool + */ size_t get_transactions_count() const; + + /** + * @brief get a string containing human-readable pool information + * + * @param short_format whether to use a shortened format for the info + * + * @return the string + */ std::string print_pool(bool short_format) const; + + /** + * @brief remove transactions from the pool which are no longer valid + * + * With new versions of the currency, what conditions render a transaction + * invalid may change. This function clears those which were received + * before a version change and no longer conform to requirements. + * + * @param version the version the transactions must conform to + * + * @return the number of transactions removed + */ size_t validate(uint8_t version); - /*bool flush_pool(const std::strig& folder); - bool inflate_pool(const std::strig& folder);*/ #define CURRENT_MEMPOOL_ARCHIVE_VER 11 +#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 11 + /** + * @brief serialize the transaction pool to/from disk + * + * If the archive version passed is older than the version compiled + * in, this function does nothing, as it cannot deserialize after a + * format change. + * + * @tparam archive_t the archive class + * @param a the archive to serialize to/from + * @param version the archive version + */ template<class archive_t> void serialize(archive_t & a, const unsigned int version) { @@ -123,97 +334,169 @@ namespace cryptonote a & m_timed_out_transactions; } + /** + * @brief information about a single transaction + */ struct tx_details { - transaction tx; - size_t blob_size; - uint64_t fee; - crypto::hash max_used_block_id; - uint64_t max_used_block_height; - bool kept_by_block; - // - uint64_t last_failed_height; + transaction tx; //!< the transaction + size_t blob_size; //!< the transaction's size + uint64_t fee; //!< the transaction's fee amount + crypto::hash max_used_block_id; //!< the hash of the highest block referenced by an input + uint64_t max_used_block_height; //!< the height of the highest block referenced by an input + + //! whether or not the transaction has been in a block before + /*! if the transaction was returned to the pool from the blockchain + * due to a reorg, then this will be true + */ + bool kept_by_block; + + //! the highest block the transaction referenced when last checking it failed + /*! if verifying a transaction's inputs fails, it's possible this is due + * to a reorg since it was created (if it used recently created outputs + * as inputs). + */ + uint64_t last_failed_height; + + //! the hash of the highest block the transaction referenced when last checking it failed + /*! if verifying a transaction's inputs fails, it's possible this is due + * to a reorg since it was created (if it used recently created outputs + * as inputs). + */ crypto::hash last_failed_id; - time_t receive_time; - time_t last_relayed_time; - bool relayed; + time_t receive_time; //!< the time when the transaction entered the pool + + time_t last_relayed_time; //!< the last time the transaction was relayed to the network + bool relayed; //!< whether or not the transaction has been relayed to the network }; private: + + /** + * @brief remove old transactions from the pool + * + * After a certain time, it is assumed that a transaction which has not + * yet been mined will likely not be mined. These transactions are removed + * from the pool to avoid buildup. + * + * @return true + */ bool remove_stuck_transactions(); + + /** + * @brief check if a transaction in the pool has a given spent key image + * + * @param key_im the spent key image to look for + * + * @return true if the spent key image is present, otherwise false + */ bool have_tx_keyimg_as_spent(const crypto::key_image& key_im) const; + + /** + * @brief check if any spent key image in a transaction is in the pool + * + * Checks if any of the spent key images in a given transaction are present + * in any of the transactions in the transaction pool. + * + * @note see tx_pool::have_tx_keyimg_as_spent + * + * @param tx the transaction to check spent key images of + * + * @return true if any spent key images are present in the pool, otherwise false + */ bool have_tx_keyimges_as_spent(const transaction& tx) const; + + /** + * @brief forget a transaction's spent key images + * + * Spent key images are stored separately from transactions for + * convenience/speed, so this is part of the process of removing + * a transaction from the pool. + * + * @param tx the transaction + * + * @return false if any key images to be removed cannot be found, otherwise true + */ bool remove_transaction_keyimages(const transaction& tx); + + /** + * @brief check if any of a transaction's spent key images are present in a given set + * + * @param kic the set of key images to check against + * @param tx the transaction to check + * + * @return true if any key images present in the set, otherwise false + */ static bool have_key_images(const std::unordered_set<crypto::key_image>& kic, const transaction& tx); + + /** + * @brief append the key images from a transaction to the given set + * + * @param kic the set of key images to append to + * @param tx the transaction + * + * @return false if any append fails, otherwise true + */ static bool append_key_images(std::unordered_set<crypto::key_image>& kic, const transaction& tx); + /** + * @brief check if a transaction is a valid candidate for inclusion in a block + * + * @param txd the transaction to check (and info about it) + * + * @return true if the transaction is good to go, otherwise false + */ bool is_transaction_ready_to_go(tx_details& txd) const; + + //! map transactions (and related info) by their hashes typedef std::unordered_map<crypto::hash, tx_details > transactions_container; + + //TODO: confirm the below comments and investigate whether or not this + // is the desired behavior + //! map key images to transactions which spent them + /*! this seems odd, but it seems that multiple transactions can exist + * in the pool which both have the same spent key. This would happen + * in the event of a reorg where someone creates a new/different + * transaction on the assumption that the original will not be in a + * block again. + */ typedef std::unordered_map<crypto::key_image, std::unordered_set<crypto::hash> > key_images_container; - mutable epee::critical_section m_transactions_lock; - transactions_container m_transactions; - key_images_container m_spent_key_images; + mutable epee::critical_section m_transactions_lock; //!< lock for the pool + transactions_container m_transactions; //!< container for transactions in the pool + + //! container for spent key images from the transactions in the pool + key_images_container m_spent_key_images; + + //TODO: this time should be a named constant somewhere, not hard-coded + //! interval on which to check for stale/"stuck" transactions epee::math_helper::once_a_time_seconds<30> m_remove_stuck_tx_interval; - //TODO: add fee_per_kb element to type tx_details and replace this - //functionality by just making m_transactions a std::set - sorted_tx_container m_txs_by_fee; + //TODO: look into doing this better + sorted_tx_container m_txs_by_fee; //!< container for transactions organized by fee per size + /** + * @brief get an iterator to a transaction in the sorted container + * + * @param id the hash of the transaction to look for + * + * @return an iterator, possibly to the end of the container if not found + */ sorted_tx_container::iterator find_tx_in_sorted_container(const crypto::hash& id) const; + //! transactions which are unlikely to be included in blocks + /*! These transactions are kept in RAM in case they *are* included + * in a block eventually, but this container is not saved to disk. + */ std::unordered_set<crypto::hash> m_timed_out_transactions; - //transactions_container m_alternative_transactions; - - std::string m_config_folder; + std::string m_config_folder; //!< the folder to save state to #if BLOCKCHAIN_DB == DB_LMDB - Blockchain& m_blockchain; + Blockchain& m_blockchain; //!< reference to the Blockchain object #else blockchain_storage& m_blockchain; #endif - /************************************************************************/ - /* */ - /************************************************************************/ - /*class inputs_visitor: public boost::static_visitor<bool> - { - key_images_container& m_spent_keys; - public: - inputs_visitor(key_images_container& spent_keys): m_spent_keys(spent_keys) - {} - bool operator()(const txin_to_key& tx) const - { - auto pr = m_spent_keys.insert(tx.k_image); - CHECK_AND_ASSERT_MES(pr.second, false, "Tried to insert transaction with input seems already spent, input: " << epee::string_tools::pod_to_hex(tx.k_image)); - return true; - } - bool operator()(const txin_gen& tx) const - { - CHECK_AND_ASSERT_MES(false, false, "coinbase transaction in memory pool"); - return false; - } - bool operator()(const txin_to_script& tx) const {return false;} - bool operator()(const txin_to_scripthash& tx) const {return false;} - }; */ - /************************************************************************/ - /* */ - /************************************************************************/ - class amount_visitor: public boost::static_visitor<uint64_t> - { - public: - uint64_t operator()(const txin_to_key& tx) const - { - return tx.amount; - } - uint64_t operator()(const txin_gen& tx) const - { - CHECK_AND_ASSERT_MES(false, false, "coinbase transaction in memory pool"); - return 0; - } - uint64_t operator()(const txin_to_script& tx) const {return 0;} - uint64_t operator()(const txin_to_scripthash& tx) const {return 0;} - }; #if BLOCKCHAIN_DB == DB_LMDB #else @@ -248,6 +531,7 @@ namespace boost } } BOOST_CLASS_VERSION(cryptonote::tx_memory_pool, CURRENT_MEMPOOL_ARCHIVE_VER) +BOOST_CLASS_VERSION(cryptonote::tx_memory_pool::tx_details, CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER) diff --git a/src/cryptonote_core/verification_context.h b/src/cryptonote_core/verification_context.h index fcfd2a3e2..e58291ea9 100644 --- a/src/cryptonote_core/verification_context.h +++ b/src/cryptonote_core/verification_context.h @@ -40,6 +40,13 @@ namespace cryptonote bool m_verifivation_failed; //bad tx, should drop connection bool m_verifivation_impossible; //the transaction is related with an alternative blockchain bool m_added_to_pool; + bool m_low_mixin; + bool m_double_spend; + bool m_invalid_input; + bool m_invalid_output; + bool m_too_big; + bool m_overspend; + bool m_fee_too_low; }; struct block_verification_context diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 2dca5e3e7..1fe2ddcf6 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -59,6 +59,11 @@ namespace daemon_args "os-version" , "OS for which this executable was compiled" }; + const command_line::arg_descriptor<unsigned> arg_max_concurrency = { + "max-concurrency" + , "Max number of threads to use for a parallel job" + , 0 + }; } // namespace daemon_args #endif // DAEMON_COMMAND_LINE_ARGS_H diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 8684b5d09..2e2f8936c 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -82,6 +82,7 @@ int main(int argc, char const * argv[]) bf::path default_log = default_data_dir / std::string(CRYPTONOTE_NAME ".log"); command_line::add_arg(core_settings, daemon_args::arg_log_file, default_log.string()); command_line::add_arg(core_settings, daemon_args::arg_log_level); + command_line::add_arg(core_settings, daemon_args::arg_max_concurrency); daemonizer::init_options(hidden_options, visible_options); daemonize::t_executor::init_options(core_settings); @@ -262,6 +263,9 @@ int main(int argc, char const * argv[]) tools::set_stack_trace_log(log_file_path.filename().string()); } + if (command_line::has_arg(vm, daemon_args::arg_max_concurrency)) + tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency)); + _note_c("dbg/main", "Moving from main() into the daemonize now."); return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm); diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 933c93ed7..ae9492111 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -70,6 +70,23 @@ namespace { << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl << "reward: " << boost::lexical_cast<std::string>(header.reward); } + + std::string get_human_time_ago(time_t t, time_t now) + { + if (t == now) + return "now"; + time_t dt = t > now ? t - now : now - t; + std::string s; + if (dt < 90) + s = boost::lexical_cast<std::string>(dt) + " seconds"; + else if (dt < 90 * 60) + s = boost::lexical_cast<std::string>(dt/60) + " minutes"; + else if (dt < 36 * 3600) + s = boost::lexical_cast<std::string>(dt/3600) + " hours"; + else + s = boost::lexical_cast<std::string>(dt/(3600*24)) + " days"; + return s + " " + (t > now ? "in the future" : "ago"); + } } t_rpc_command_executor::t_rpc_command_executor( @@ -575,16 +592,26 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash) { } } - if (1 == res.txs_as_hex.size()) + if (1 == res.txs.size() || 1 == res.txs_as_hex.size()) { + if (1 == res.txs.size()) + { + // only available for new style answers + if (res.txs.front().in_pool) + tools::success_msg_writer() << "Found in pool"; + else + tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height; + } + // first as hex - tools::success_msg_writer() << res.txs_as_hex.front(); + const std::string &as_hex = (1 == res.txs.size()) ? res.txs.front().as_hex : res.txs_as_hex.front(); + tools::success_msg_writer() << as_hex; // then as json crypto::hash tx_hash, tx_prefix_hash; cryptonote::transaction tx; cryptonote::blobdata blob; - if (!string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), blob)) + if (!string_tools::parse_hexstr_to_binbuff(as_hex, blob)) { tools::fail_msg_writer() << "Failed to parse tx"; } @@ -669,6 +696,7 @@ bool t_rpc_command_executor::print_transaction_pool_long() { } if (! res.transactions.empty()) { + const time_t now = time(NULL); tools::msg_writer() << "Transactions: "; for (auto & tx_info : res.transactions) { @@ -676,7 +704,7 @@ bool t_rpc_command_executor::print_transaction_pool_long() { << tx_info.tx_json << std::endl << "blob_size: " << tx_info.blob_size << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "receive_time: " << tx_info.receive_time << std::endl + << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl << "max_used_block_height: " << tx_info.max_used_block_height << std::endl << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl @@ -747,17 +775,21 @@ bool t_rpc_command_executor::print_transaction_pool_short() { { tools::msg_writer() << "Pool is empty" << std::endl; } - for (auto & tx_info : res.transactions) + else { - tools::msg_writer() << "id: " << tx_info.id_hash << std::endl - << "blob_size: " << tx_info.blob_size << std::endl - << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "receive_time: " << tx_info.receive_time << std::endl - << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl - << "max_used_block_height: " << tx_info.max_used_block_height << std::endl - << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl - << "last_failed_height: " << tx_info.last_failed_height << std::endl - << "last_failed_id: " << tx_info.last_failed_id_hash << std::endl; + const time_t now = time(NULL); + for (auto & tx_info : res.transactions) + { + tools::msg_writer() << "id: " << tx_info.id_hash << std::endl + << "blob_size: " << tx_info.blob_size << std::endl + << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl + << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl + << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl + << "max_used_block_height: " << tx_info.max_used_block_height << std::endl + << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl + << "last_failed_height: " << tx_info.last_failed_height << std::endl + << "last_failed_id: " << tx_info.last_failed_id_hash << std::endl; + } } return true; diff --git a/src/p2p/connection_basic.cpp b/src/p2p/connection_basic.cpp index df702134e..981a02882 100644 --- a/src/p2p/connection_basic.cpp +++ b/src/p2p/connection_basic.cpp @@ -167,7 +167,7 @@ connection_basic::connection_basic(boost::asio::io_service& io_service, std::ato //boost::filesystem::create_directories("log/dr-monero/net/"); } -connection_basic::~connection_basic() { +connection_basic::~connection_basic() noexcept(false) { string remote_addr_str = "?"; m_ref_sock_count--; try { boost::system::error_code e; remote_addr_str = socket_.remote_endpoint(e).address().to_string(); } catch(...){} ; diff --git a/src/p2p/connection_basic.hpp b/src/p2p/connection_basic.hpp index 8be798735..5452fa6fb 100644 --- a/src/p2p/connection_basic.hpp +++ b/src/p2p/connection_basic.hpp @@ -103,7 +103,7 @@ class connection_basic { // not-templated base class for rapid developmet of som // first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number); - virtual ~connection_basic(); + virtual ~connection_basic() noexcept(false); // various handlers to be called from connection class: void do_send_handler_write(const void * ptr , size_t cb); diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp index ca0726c5f..fe54aef63 100644 --- a/src/p2p/data_logger.cpp +++ b/src/p2p/data_logger.cpp @@ -103,7 +103,7 @@ namespace net_utils _info_c("dbg/data","Data logger constructed"); } - data_logger::~data_logger() { + data_logger::~data_logger() noexcept(false) { _note_c("dbg/data","Destructor of the data logger"); { boost::lock_guard<boost::mutex> lock(mMutex); diff --git a/src/p2p/data_logger.hpp b/src/p2p/data_logger.hpp index 148afc0ab..278d08bfc 100644 --- a/src/p2p/data_logger.hpp +++ b/src/p2p/data_logger.hpp @@ -55,7 +55,7 @@ enum class data_logger_state { state_before_init, state_during_init, state_ready public: static data_logger &get_instance(); ///< singleton static void kill_instance(); ///< call this before ending main to allow more gracefull shutdown of the main singleton and it's background thread - ~data_logger(); ///< destr, will be called when singleton is killed when global m_obj dies. will kill theads etc + ~data_logger() noexcept(false); ///< destr, will be called when singleton is killed when global m_obj dies. will kill theads etc private: data_logger(); ///< constructor is private, use only via singleton get_instance diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 6278db891..0fab40322 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1319,6 +1319,7 @@ namespace nodetool if(rsp.status != PING_OK_RESPONSE_STATUS_TEXT || pr != rsp.peer_id) { LOG_PRINT_CC_L2(ping_context, "back ping invoke wrong response \"" << rsp.status << "\" from" << ip << ":" << port << ", hsh_peer_id=" << pr_ << ", rsp.peer_id=" << rsp.peer_id); + m_net_server.get_config_object().close(ping_context.m_connection_id); return; } m_net_server.get_config_object().close(ping_context.m_connection_id); diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 5350b6fe0..cfe1057ac 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -166,6 +166,25 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res) + { + CHECK_CORE_BUSY(); + NOTIFY_RESPONSE_CHAIN_ENTRY::request resp; + + resp.start_height = req.start_height; + if(!m_core.find_blockchain_supplement(req.block_ids, resp)) + { + res.status = "Failed"; + return false; + } + res.current_height = resp.total_height; + res.start_height = resp.start_height; + res.m_block_ids = std::move(resp.m_block_ids); + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) { CHECK_CORE_BUSY(); @@ -240,6 +259,7 @@ namespace cryptonote // try the pool for any missing txes size_t found_in_pool = 0; + std::unordered_set<crypto::hash> pool_tx_hashes; if (!missed_txs.empty()) { std::list<transaction> pool_txs; @@ -248,9 +268,11 @@ namespace cryptonote { for (std::list<transaction>::const_iterator i = pool_txs.begin(); i != pool_txs.end(); ++i) { - std::list<crypto::hash>::iterator mi = std::find(missed_txs.begin(), missed_txs.end(), get_transaction_hash(*i)); + crypto::hash tx_hash = get_transaction_hash(*i); + std::list<crypto::hash>::iterator mi = std::find(missed_txs.begin(), missed_txs.end(), tx_hash); if (mi != missed_txs.end()) { + pool_tx_hashes.insert(tx_hash); missed_txs.erase(mi); txs.push_back(*i); ++found_in_pool; @@ -260,12 +282,33 @@ namespace cryptonote LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool"); } + std::list<std::string>::const_iterator txhi = req.txs_hashes.begin(); + std::vector<crypto::hash>::const_iterator vhi = vh.begin(); BOOST_FOREACH(auto& tx, txs) { + res.txs.push_back(COMMAND_RPC_GET_TRANSACTIONS::entry()); + COMMAND_RPC_GET_TRANSACTIONS::entry &e = res.txs.back(); + + crypto::hash tx_hash = *vhi++; + e.tx_hash = *txhi++; blobdata blob = t_serializable_object_to_blob(tx); - res.txs_as_hex.push_back(string_tools::buff_to_hex_nodelimer(blob)); + e.as_hex = string_tools::buff_to_hex_nodelimer(blob); if (req.decode_as_json) - res.txs_as_json.push_back(obj_to_json_str(tx)); + e.as_json = obj_to_json_str(tx); + e.in_pool = pool_tx_hashes.find(tx_hash) != pool_tx_hashes.end(); + if (e.in_pool) + { + e.block_height = std::numeric_limits<uint64_t>::max(); + } + else + { + e.block_height = m_core.get_blockchain_storage().get_db().get_tx_block_height(tx_hash); + } + + // fill up old style responses too, in case an old wallet asks + res.txs_as_hex.push_back(e.as_hex); + if (req.decode_as_json) + res.txs_as_json.push_back(e.as_json); } BOOST_FOREACH(const auto& miss_tx, missed_txs) @@ -273,7 +316,7 @@ namespace cryptonote res.missed_tx.push_back(string_tools::pod_to_hex(miss_tx)); } - LOG_PRINT_L2(res.txs_as_hex.size() << " transactions found, " << res.missed_tx.size() << " not found"); + LOG_PRINT_L2(res.txs.size() << " transactions found, " << res.missed_tx.size() << " not found"); res.status = CORE_RPC_STATUS_OK; return true; } @@ -355,24 +398,40 @@ namespace cryptonote cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); - if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false)) + if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false) || tvc.m_verifivation_failed) { - LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx"); - res.status = "Failed"; - return true; - } - - if(tvc.m_verifivation_failed) - { - LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed"); + if (tvc.m_verifivation_failed) + { + LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed"); + } + else + { + LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx"); + } res.status = "Failed"; + if ((res.low_mixin = tvc.m_low_mixin)) + res.reason = "mixin too low"; + if ((res.double_spend = tvc.m_double_spend)) + res.reason = "double spend"; + if ((res.invalid_input = tvc.m_invalid_input)) + res.reason = "invalid input"; + if ((res.invalid_output = tvc.m_invalid_output)) + res.reason = "invalid output"; + if ((res.too_big = tvc.m_too_big)) + res.reason = "too big"; + if ((res.overspend = tvc.m_overspend)) + res.reason = "overspend"; + if ((res.fee_too_low = tvc.m_fee_too_low)) + res.reason = "fee too low"; return true; } - if(!tvc.m_should_be_relayed) + if(!tvc.m_should_be_relayed || req.do_not_relay) { LOG_PRINT_L0("[on_send_raw_tx]: tx accepted, but not relayed"); - res.status = "Not relayed"; + res.reason = "Not relayed"; + res.not_relayed = true; + res.status = CORE_RPC_STATUS_OK; return true; } @@ -627,8 +686,10 @@ namespace cryptonote LOG_ERROR("Failed to calculate offset for "); return false; } + blobdata hashing_blob = get_block_hashing_blob(b); res.prev_hash = string_tools::pod_to_hex(b.prev_id); res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); + res.blockhashing_blob = string_tools::buff_to_hex_nodelimer(hashing_blob); res.status = CORE_RPC_STATUS_OK; return true; } @@ -869,7 +930,7 @@ namespace cryptonote { res.tx_hashes.push_back(epee::string_tools::pod_to_hex(blk.tx_hashes[n])); } - res.blob = t_serializable_object_to_blob(blk); + res.blob = string_tools::buff_to_hex_nodelimer(t_serializable_object_to_blob(blk)); res.json = obj_to_json_str(blk); res.status = CORE_RPC_STATUS_OK; return true; diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 5c3707209..a7a54b59f 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -75,6 +75,7 @@ namespace cryptonote BEGIN_URI_MAP2() MAP_URI_AUTO_JON2("/getheight", on_get_height, COMMAND_RPC_GET_HEIGHT) MAP_URI_AUTO_BIN2("/getblocks.bin", on_get_blocks, COMMAND_RPC_GET_BLOCKS_FAST) + MAP_URI_AUTO_BIN2("/gethashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/get_o_indexes.bin", on_get_indexes, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES) MAP_URI_AUTO_BIN2("/getrandom_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) MAP_URI_AUTO_JON2("/gettransactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) @@ -103,18 +104,19 @@ namespace cryptonote MAP_JON_RPC_WE("getblockheaderbyhash", on_get_block_header_by_hash, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH) MAP_JON_RPC_WE("getblockheaderbyheight", on_get_block_header_by_height, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT) MAP_JON_RPC_WE("getblock", on_get_block, COMMAND_RPC_GET_BLOCK) - MAP_JON_RPC_WE("get_connections", on_get_connections, COMMAND_RPC_GET_CONNECTIONS) + MAP_JON_RPC_WE_IF("get_connections", on_get_connections, COMMAND_RPC_GET_CONNECTIONS, !m_restricted) MAP_JON_RPC_WE("get_info", on_get_info_json, COMMAND_RPC_GET_INFO) MAP_JON_RPC_WE("hard_fork_info", on_hard_fork_info, COMMAND_RPC_HARD_FORK_INFO) - MAP_JON_RPC_WE("setbans", on_set_bans, COMMAND_RPC_SETBANS) - MAP_JON_RPC_WE("getbans", on_get_bans, COMMAND_RPC_GETBANS) - MAP_JON_RPC_WE("flush_txpool", on_flush_txpool, COMMAND_RPC_FLUSH_TRANSACTION_POOL) + MAP_JON_RPC_WE_IF("setbans", on_set_bans, COMMAND_RPC_SETBANS, !m_restricted) + MAP_JON_RPC_WE_IF("getbans", on_get_bans, COMMAND_RPC_GETBANS, !m_restricted) + MAP_JON_RPC_WE_IF("flush_txpool", on_flush_txpool, COMMAND_RPC_FLUSH_TRANSACTION_POOL, !m_restricted) MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) END_JSON_RPC_MAP() END_URI_MAP2() bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res); bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res); + bool on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res); bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res); bool on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res); bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 6d4dd1252..392c7501f 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -89,6 +89,36 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; + + struct COMMAND_RPC_GET_HASHES_FAST + { + + struct request + { + std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ + uint64_t start_height; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) + KV_SERIALIZE(start_height) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::list<crypto::hash> m_block_ids; + uint64_t start_height; + uint64_t current_height; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids) + KV_SERIALIZE(start_height) + KV_SERIALIZE(current_height) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + //----------------------------------------------- struct COMMAND_RPC_GET_TRANSACTIONS { @@ -103,18 +133,41 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; + struct entry + { + std::string tx_hash; + std::string as_hex; + std::string as_json; + bool in_pool; + uint64_t block_height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(tx_hash) + KV_SERIALIZE(as_hex) + KV_SERIALIZE(as_json) + KV_SERIALIZE(in_pool) + KV_SERIALIZE(block_height) + END_KV_SERIALIZE_MAP() + }; struct response { - std::list<std::string> txs_as_hex; //transactions blobs as hex + // older compatibility stuff + std::list<std::string> txs_as_hex; //transactions blobs as hex (old compat) + std::list<std::string> txs_as_json; //transactions decoded as json (old compat) + + // in both old and new std::list<std::string> missed_tx; //not found transactions - std::list<std::string> txs_as_json; //transactions decoded as json + + // new style + std::vector<entry> txs; std::string status; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs_as_hex) - KV_SERIALIZE(missed_tx) KV_SERIALIZE(txs_as_json) + KV_SERIALIZE(txs) + KV_SERIALIZE(missed_tx) KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; @@ -221,12 +274,14 @@ namespace cryptonote struct request { std::string tx_as_hex; + bool do_not_relay; request() {} explicit request(const transaction &); BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_as_hex) + KV_SERIALIZE(do_not_relay) END_KV_SERIALIZE_MAP() }; @@ -234,9 +289,27 @@ namespace cryptonote struct response { std::string status; + std::string reason; + bool not_relayed; + bool low_mixin; + bool double_spend; + bool invalid_input; + bool invalid_output; + bool too_big; + bool overspend; + bool fee_too_low; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) + KV_SERIALIZE(reason) + KV_SERIALIZE(not_relayed) + KV_SERIALIZE(low_mixin) + KV_SERIALIZE(double_spend) + KV_SERIALIZE(invalid_input) + KV_SERIALIZE(invalid_output) + KV_SERIALIZE(too_big) + KV_SERIALIZE(overspend) + KV_SERIALIZE(fee_too_low) END_KV_SERIALIZE_MAP() }; }; @@ -427,6 +500,7 @@ namespace cryptonote uint64_t reserved_offset; std::string prev_hash; blobdata blocktemplate_blob; + blobdata blockhashing_blob; std::string status; BEGIN_KV_SERIALIZE_MAP() @@ -435,6 +509,7 @@ namespace cryptonote KV_SERIALIZE(reserved_offset) KV_SERIALIZE(prev_hash) KV_SERIALIZE(blocktemplate_blob) + KV_SERIALIZE(blockhashing_blob) KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 04170df62..7d28de9c0 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -58,6 +58,7 @@ #include "crypto/crypto.h" // for crypto::secret_key definition #include "mnemonics/electrum-words.h" #include "rapidjson/document.h" +#include "common/json_util.h" #include <stdexcept> #if defined(WIN32) @@ -76,6 +77,23 @@ typedef cryptonote::simple_wallet sw; #define DEFAULT_MIX 4 +#define LOCK_REFRESH_THREAD_SCOPE() \ + bool auto_refresh_run = m_auto_refresh_run.load(std::memory_order_relaxed); \ + m_auto_refresh_run.store(false, std::memory_order_relaxed); \ + /* stop any background refresh, and take over */ \ + m_wallet->stop(); \ + m_auto_refresh_mutex.lock(); \ + m_auto_refresh_cond.notify_one(); \ + m_auto_refresh_mutex.unlock(); \ + if (auto_refresh_run) \ + m_auto_refresh_thread.join(); \ + boost::unique_lock<boost::mutex> lock(m_auto_refresh_mutex); \ + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ \ + m_auto_refresh_run.store(auto_refresh_run, std::memory_order_relaxed); \ + if (auto_refresh_run) \ + m_auto_refresh_thread = boost::thread([&]{wallet_refresh_thread();}); \ + }) + namespace { const command_line::arg_descriptor<std::string> arg_wallet_file = {"wallet-file", sw::tr("Use wallet <arg>"), ""}; @@ -92,10 +110,12 @@ namespace const command_line::arg_descriptor<bool> arg_non_deterministic = {"non-deterministic", sw::tr("Create non-deterministic view and spend keys"), false}; const command_line::arg_descriptor<int> arg_daemon_port = {"daemon-port", sw::tr("Use daemon instance at port <arg> instead of 18081"), 0}; const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", LOG_LEVEL_0}; + const command_line::arg_descriptor<uint32_t> arg_max_concurrency = {"max-concurrency", "Max number of threads to use for a parallel job", 0}; const command_line::arg_descriptor<std::string> arg_log_file = {"log-file", sw::tr("Specify log file"), ""}; const command_line::arg_descriptor<bool> arg_testnet = {"testnet", sw::tr("For testnet. Daemon must also be launched with --testnet flag"), false}; const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", sw::tr("Restricts RPC to view-only commands"), false}; const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", sw::tr("Enable commands which rely on a trusted daemon"), false}; + const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0}; const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; @@ -212,6 +232,44 @@ namespace return true; return false; } + + const struct + { + const char *name; + tools::wallet2::RefreshType refresh_type; + } refresh_type_names[] = + { + { "full", tools::wallet2::RefreshFull }, + { "optimize-coinbase", tools::wallet2::RefreshOptimizeCoinbase }, + { "optimized-coinbase", tools::wallet2::RefreshOptimizeCoinbase }, + { "no-coinbase", tools::wallet2::RefreshNoCoinbase }, + { "default", tools::wallet2::RefreshDefault }, + }; + + bool parse_refresh_type(const std::string &s, tools::wallet2::RefreshType &refresh_type) + { + for (size_t n = 0; n < sizeof(refresh_type_names) / sizeof(refresh_type_names[0]); ++n) + { + if (s == refresh_type_names[n].name) + { + refresh_type = refresh_type_names[n].refresh_type; + return true; + } + } + fail_msg_writer() << tr("failed to parse refresh type"); + return false; + } + + std::string get_refresh_type_name(tools::wallet2::RefreshType type) + { + for (size_t n = 0; n < sizeof(refresh_type_names) / sizeof(refresh_type_names[0]); ++n) + { + if (type == refresh_type_names[n].refresh_type) + return refresh_type_names[n].name; + } + return "invalid"; + } + } @@ -257,6 +315,8 @@ bool simple_wallet::seed(const std::vector<std::string> &args/* = std::vector<st if (m_wallet->get_seed_language().empty()) { std::string mnemonic_language = get_mnemonic_language(); + if (mnemonic_language.empty()) + return true; m_wallet->set_seed_language(mnemonic_language); } @@ -304,6 +364,8 @@ bool simple_wallet::seed_set_language(const std::vector<std::string> &args/* = s } std::string mnemonic_language = get_mnemonic_language(); + if (mnemonic_language.empty()) + return true; m_wallet->set_seed_language(mnemonic_language); m_wallet->rewrite(m_wallet_file, pwd_container.password()); return true; @@ -460,32 +522,6 @@ bool simple_wallet::set_auto_refresh(const std::vector<std::string> &args/* = st return true; } -static bool parse_refresh_type(const std::string &s, tools::wallet2::RefreshType &refresh_type) -{ - static const struct - { - const char *name; - tools::wallet2::RefreshType refresh_type; - } names[] = - { - { "full", tools::wallet2::RefreshFull }, - { "optimize-coinbase", tools::wallet2::RefreshOptimizeCoinbase }, - { "optimized-coinbase", tools::wallet2::RefreshOptimizeCoinbase }, - { "no-coinbase", tools::wallet2::RefreshNoCoinbase }, - { "default", tools::wallet2::RefreshDefault }, - }; - for (size_t n = 0; n < sizeof(names) / sizeof(names[0]); ++n) - { - if (s == names[n].name) - { - refresh_type = names[n].refresh_type; - return true; - } - } - fail_msg_writer() << tr("failed to parse refresh type"); - return false; -} - bool simple_wallet::set_refresh_type(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { bool success = false; @@ -542,6 +578,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), tr("transfer [<mixin_count>] <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of extra inputs to include for untraceability (from 0 to maximum available)")); m_cmd_binder.set_handler("transfer_new", boost::bind(&simple_wallet::transfer_new, this, _1), tr("Same as transfer, but using a new transaction building algorithm")); m_cmd_binder.set_handler("sweep_unmixable", boost::bind(&simple_wallet::sweep_unmixable, this, _1), tr("Send all unmixable outputs to yourself with mixin 0")); + m_cmd_binder.set_handler("sweep_all", boost::bind(&simple_wallet::sweep_all, this, _1), tr("Send all unlocked balance an address")); m_cmd_binder.set_handler("set_log", boost::bind(&simple_wallet::set_log, this, _1), tr("set_log <level> - Change current log detail level, <0-4>")); m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), tr("Show current wallet public address")); m_cmd_binder.set_handler("integrated_address", boost::bind(&simple_wallet::print_integrated_address, this, _1), tr("integrated_address [PID] - Encode a payment ID into an integrated address for the current wallet public address (no argument uses a random payment ID), or decode an integrated address to standard address and payment ID")); @@ -556,6 +593,8 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("check_tx_key", boost::bind(&simple_wallet::check_tx_key, this, _1), tr("Check amount going to <address> in <txid>")); m_cmd_binder.set_handler("show_transfers", boost::bind(&simple_wallet::show_transfers, this, _1), tr("show_transfers [in|out] [<min_height> [<max_height>]] - Show incoming/outgoing transfers within an optional height range")); m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), tr("Rescan blockchain from scratch")); + m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), tr("Set an arbitrary string note for a txid")); + m_cmd_binder.set_handler("get_tx_note", boost::bind(&simple_wallet::get_tx_note, this, _1), tr("Get a string note for a txid")); m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::help, this, _1), tr("Show this help")); } //---------------------------------------------------------------------------------------------------- @@ -563,7 +602,12 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) { if (args.empty()) { - fail_msg_writer() << tr("set: needs an argument. available options: seed, always-confirm-transfers, default-mixin, auto-refresh, refresh-type"); + success_msg_writer() << "seed = " << m_wallet->get_seed_language(); + success_msg_writer() << "always-confirm-transfers = " << m_wallet->always_confirm_transfers(); + success_msg_writer() << "store-tx-info = " << m_wallet->store_tx_info(); + success_msg_writer() << "default-mixin = " << m_wallet->default_mixin(); + success_msg_writer() << "auto-refresh = " << m_wallet->auto_refresh(); + success_msg_writer() << "refresh-type = " << get_refresh_type_name(m_wallet->get_refresh_type()); return true; } else @@ -698,6 +742,10 @@ bool simple_wallet::ask_wallet_create_if_needed() tr("Specify wallet file name (e.g., MyWallet). If the wallet doesn't exist, it will be created.\n" "Wallet file name: ") ); + if (std::cin.eof()) + { + return false; + } valid_path = tools::wallet2::wallet_valid_path_format(wallet_path); if (!valid_path) { @@ -820,6 +868,8 @@ static bool get_password(const boost::program_options::variables_map& vm, bool a //---------------------------------------------------------------------------------------------------- bool simple_wallet::generate_from_json(const boost::program_options::variables_map& vm, std::string &wallet_file, std::string &password) { + bool testnet = command_line::get_arg(vm, arg_testnet); + std::string buf; bool r = epee::file_io_utils::load_file_to_string(m_generate_from_json, buf); if (!r) { @@ -833,84 +883,123 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m return false; } - if (!json.HasMember("version")) { - fail_msg_writer() << tr("Version not found in JSON"); - return false; - } - unsigned int version = json["version"].GetUint(); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, version, unsigned, Uint, true); const int current_version = 1; - if (version > current_version) { - fail_msg_writer() << boost::format(tr("Version %u too new, we can only grok up to %u")) % version % current_version; - return false; - } - if (!json.HasMember("filename")) { - fail_msg_writer() << tr("Filename not found in JSON"); + if (field_version > current_version) { + fail_msg_writer() << boost::format(tr("Version %u too new, we can only grok up to %u")) % field_version % current_version; return false; } - std::string filename = json["filename"].GetString(); - bool recover = false; - uint64_t scan_from_height = 0; - if (json.HasMember("scan_from_height")) { - scan_from_height = json["scan_from_height"].GetUint64(); - recover = true; - } + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, filename, std::string, String, true); - password = ""; - if (json.HasMember("password")) { - password = json["password"].GetString(); - } + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, scan_from_height, uint64_t, Uint64, false); + bool recover = field_scan_from_height_found; - std::string viewkey_string(""); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, password, std::string, String, false); + password = field_password; + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, viewkey, std::string, String, false); crypto::secret_key viewkey; - if (json.HasMember("viewkey")) { - viewkey_string = json["viewkey"].GetString(); + if (field_viewkey_found) + { cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data)) + if(!epee::string_tools::parse_hexstr_to_binbuff(field_viewkey, viewkey_data)) { fail_msg_writer() << tr("failed to parse view key secret key"); return false; } viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); + crypto::public_key pkey; + if (!crypto::secret_key_to_public_key(viewkey, pkey)) { + fail_msg_writer() << tr("failed to verify view key secret key"); + return false; + } } - std::string spendkey_string(""); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, spendkey, std::string, String, false); crypto::secret_key spendkey; - if (json.HasMember("spendkey")) { - spendkey_string = json["spendkey"].GetString(); + if (field_spendkey_found) + { cryptonote::blobdata spendkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(spendkey_string, spendkey_data)) + if(!epee::string_tools::parse_hexstr_to_binbuff(field_spendkey, spendkey_data)) { fail_msg_writer() << tr("failed to parse spend key secret key"); return false; } spendkey = *reinterpret_cast<const crypto::secret_key*>(spendkey_data.data()); + crypto::public_key pkey; + if (!crypto::secret_key_to_public_key(spendkey, pkey)) { + fail_msg_writer() << tr("failed to verify spend key secret key"); + return false; + } } - std::string seed(""); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, seed, std::string, String, false); std::string old_language; - if (json.HasMember("seed")) { - seed = json["seed"].GetString(); - if (!crypto::ElectrumWords::words_to_bytes(seed, m_recovery_key, old_language)) + if (field_seed_found) + { + if (!crypto::ElectrumWords::words_to_bytes(field_seed, m_recovery_key, old_language)) { fail_msg_writer() << tr("Electrum-style word list failed verification"); return false; } - m_electrum_seed = seed; + m_electrum_seed = field_seed; m_restore_deterministic_wallet = true; } + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, address, std::string, String, false); + // compatibility checks - if (seed.empty() && viewkey_string.empty()) { + if (!field_seed_found && !field_viewkey_found) + { fail_msg_writer() << tr("At least one of Electrum-style word list and private view key must be specified"); return false; } - if (!seed.empty() && (!viewkey_string.empty() || !spendkey_string.empty())) { + if (field_seed_found && (field_viewkey_found || field_spendkey_found)) + { fail_msg_writer() << tr("Both Electrum-style word list and private key(s) specified"); return false; } - m_wallet_file = filename; + // if an address was given, we check keys against it, and deduce the spend + // public key if it was not given + if (field_address_found) + { + cryptonote::account_public_address address; + bool has_payment_id; + crypto::hash8 new_payment_id; + if(!get_account_integrated_address_from_str(address, has_payment_id, new_payment_id, testnet, field_address)) + { + fail_msg_writer() << tr("invalid address"); + return false; + } + if (field_viewkey_found) + { + crypto::public_key pkey; + if (!crypto::secret_key_to_public_key(viewkey, pkey)) { + fail_msg_writer() << tr("failed to verify view key secret key"); + return false; + } + if (address.m_view_public_key != pkey) { + fail_msg_writer() << tr("view key does not match standard address"); + return false; + } + } + if (field_spendkey_found) + { + crypto::public_key pkey; + if (!crypto::secret_key_to_public_key(spendkey, pkey)) { + fail_msg_writer() << tr("failed to verify spend key secret key"); + return false; + } + if (address.m_spend_public_key != pkey) { + fail_msg_writer() << tr("spend key does not match standard address"); + return false; + } + } + } + + m_wallet_file = field_filename; bool was_deprecated_wallet = m_restore_deterministic_wallet && ((old_language == crypto::ElectrumWords::old_language_name) || crypto::ElectrumWords::get_is_old_style_seed(m_electrum_seed)); @@ -919,13 +1008,13 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m return false; } - bool testnet = command_line::get_arg(vm, arg_testnet); m_wallet.reset(new tools::wallet2(testnet)); m_wallet->callback(this); + m_wallet->set_refresh_from_block_height(field_scan_from_height); try { - if (!seed.empty()) + if (!field_seed.empty()) { m_wallet->generate(m_wallet_file, password, m_recovery_key, recover, false); } @@ -936,17 +1025,27 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m fail_msg_writer() << tr("failed to verify view key secret key"); return false; } - if (!crypto::secret_key_to_public_key(spendkey, address.m_spend_public_key)) { - fail_msg_writer() << tr("failed to verify spend key secret key"); - return false; - } - if (spendkey_string.empty()) + if (field_spendkey.empty()) { + // if we have an addres but no spend key, we can deduce the spend public key + // from the address + if (field_address_found) + { + cryptonote::account_public_address address2; + bool has_payment_id; + crypto::hash8 new_payment_id; + get_account_integrated_address_from_str(address2, has_payment_id, new_payment_id, testnet, field_address); + address.m_spend_public_key = address2.m_spend_public_key; + } m_wallet->generate(m_wallet_file, password, address, viewkey); } else { + if (!crypto::secret_key_to_public_key(spendkey, address.m_spend_public_key)) { + fail_msg_writer() << tr("failed to verify spend key secret key"); + return false; + } m_wallet->generate(m_wallet_file, password, address, spendkey, viewkey); } } @@ -957,13 +1056,42 @@ bool simple_wallet::generate_from_json(const boost::program_options::variables_m return false; } - m_wallet->set_refresh_from_block_height(scan_from_height); - wallet_file = m_wallet_file; return r; } +static bool is_local_daemon(const std::string &address) +{ + // extract host + epee::net_utils::http::url_content u_c; + if (!epee::net_utils::parse_url(address, u_c)) + { + LOG_PRINT_L1("Failed to determine whether daemon is local, assuming not"); + return false; + } + if (u_c.host.empty()) + { + LOG_PRINT_L1("Failed to determine whether daemon is local, assuming not"); + return false; + } + + // resolve to IP + boost::asio::io_service io_service; + boost::asio::ip::tcp::resolver resolver(io_service); + boost::asio::ip::tcp::resolver::query query(u_c.host, ""); + boost::asio::ip::tcp::resolver::iterator i = resolver.resolve(query); + while (i != boost::asio::ip::tcp::resolver::iterator()) + { + const boost::asio::ip::tcp::endpoint &ep = *i; + if (ep.address().is_loopback()) + return true; + ++i; + } + + return false; +} + //---------------------------------------------------------------------------------------------------- bool simple_wallet::init(const boost::program_options::variables_map& vm) { @@ -999,6 +1127,17 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (m_daemon_address.empty()) m_daemon_address = std::string("http://") + m_daemon_host + ":" + std::to_string(m_daemon_port); + // set --trusted-daemon if local + try + { + if (is_local_daemon(m_daemon_address)) + { + LOG_PRINT_L1(tr("Daemon is local, assuming trusted")); + m_trusted_daemon = true; + } + } + catch (const std::exception &e) { } + tools::password_container pwd_container; if (!get_password(vm, true, pwd_container)) return false; @@ -1020,6 +1159,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (m_electrum_seed.empty()) { m_electrum_seed = command_line::input_line("Specify Electrum seed: "); + if (std::cin.eof()) + return false; if (m_electrum_seed.empty()) { fail_msg_writer() << tr("specify a recovery parameter with the --electrum-seed=\"words list here\""); @@ -1033,10 +1174,28 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } } + if (!m_restore_height && m_generate_new.empty()) + { + std::string heightstr = command_line::input_line("Restore from specific blockchain height (optional, default 0): "); + if (std::cin.eof()) + return false; + if (heightstr.size()) + { + try { + m_restore_height = boost::lexical_cast<uint64_t>(heightstr); + } + catch (boost::bad_lexical_cast &) { + fail_msg_writer() << tr("bad m_restore_height parameter:") << " " << heightstr; + return false; + } + } + } if (!m_generate_from_view_key.empty()) { // parse address std::string address_string = command_line::input_line("Standard address: "); + if (std::cin.eof()) + return false; if (address_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; @@ -1052,6 +1211,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) // parse view secret key std::string viewkey_string = command_line::input_line("View key: "); + if (std::cin.eof()) + return false; if (viewkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; @@ -1084,6 +1245,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { // parse address std::string address_string = command_line::input_line("Standard address: "); + if (std::cin.eof()) + return false; if (address_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; @@ -1099,6 +1262,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) // parse spend secret key std::string spendkey_string = command_line::input_line("Spend key: "); + if (std::cin.eof()) + return false; if (spendkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; @@ -1113,6 +1278,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) // parse view secret key std::string viewkey_string = command_line::input_line("View key: "); + if (std::cin.eof()) + return false; if (viewkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; @@ -1193,6 +1360,7 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_restore_deterministic_wallet = command_line::get_arg(vm, arg_restore_deterministic_wallet); m_non_deterministic = command_line::get_arg(vm, arg_non_deterministic); m_trusted_daemon = command_line::get_arg(vm, arg_trusted_daemon); + m_restore_height = command_line::get_arg(vm, arg_restore_height); return true; } @@ -1232,6 +1400,8 @@ std::string simple_wallet::get_mnemonic_language() while (language_number < 0) { language_choice = command_line::input_line(tr("Enter the number corresponding to the language of your choice: ")); + if (std::cin.eof()) + return std::string(); try { language_number = std::stoi(language_choice); @@ -1270,6 +1440,8 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string "a deprecated version of the wallet. Please use the new seed that we provide.\n"); } mnemonic_language = get_mnemonic_language(); + if (mnemonic_language.empty()) + return false; } m_wallet_file = wallet_file; @@ -1278,6 +1450,16 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string m_wallet->callback(this); m_wallet->set_seed_language(mnemonic_language); + // for a totally new account, we don't care about older blocks. + if (!m_generate_new.empty()) + { + std::string err; + m_wallet->set_refresh_from_block_height(get_daemon_blockchain_height(err)); + } else if (m_restore_height) + { + m_wallet->set_refresh_from_block_height(m_restore_height); + } + crypto::secret_key recovery_val; try { @@ -1325,6 +1507,8 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string m_wallet.reset(new tools::wallet2(testnet)); m_wallet->callback(this); + if (m_restore_height) + m_wallet->set_refresh_from_block_height(m_restore_height); try { @@ -1351,6 +1535,8 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string m_wallet.reset(new tools::wallet2(testnet)); m_wallet->callback(this); + if (m_restore_height) + m_wallet->set_refresh_from_block_height(m_restore_height); try { @@ -1397,6 +1583,8 @@ bool simple_wallet::open_wallet(const string &wallet_file, const std::string& pa message_writer(epee::log_space::console_color_green, false) << "\n" << tr("You had been using " "a deprecated version of the wallet. Please proceed to upgrade your wallet.\n"); std::string mnemonic_language = get_mnemonic_language(); + if (mnemonic_language.empty()) + return false; m_wallet->set_seed_language(mnemonic_language); m_wallet->rewrite(m_wallet_file, password); @@ -1465,6 +1653,7 @@ bool simple_wallet::save(const std::vector<std::string> &args) { try { + LOCK_REFRESH_THREAD_SCOPE(); m_wallet->store(); success_msg_writer() << tr("Wallet data saved"); } @@ -1520,7 +1709,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) req.miner_address = m_wallet->get_account().get_public_address_str(m_wallet->testnet()); bool ok = true; - size_t max_mining_threads_count = (std::max)(boost::thread::hardware_concurrency(), static_cast<unsigned>(2)); + size_t max_mining_threads_count = (std::max)(tools::get_max_concurrency(), static_cast<unsigned>(2)); if (0 == args.size()) { req.threads_count = 1; @@ -1633,12 +1822,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset) if (!try_connect_to_daemon()) return true; - bool auto_refresh_run = m_auto_refresh_run.load(std::memory_order_relaxed); - m_auto_refresh_run.store(false, std::memory_order_relaxed); - // stop any background refresh, and take over - m_wallet->stop(); - boost::unique_lock<boost::mutex> lock(m_auto_refresh_mutex); - m_auto_refresh_cond.notify_one(); + LOCK_REFRESH_THREAD_SCOPE(); if (reset) m_wallet->rescan_blockchain(false); @@ -1651,13 +1835,13 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset) try { m_in_manual_refresh.store(true, std::memory_order_relaxed); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); m_wallet->refresh(start_height, fetched_blocks); - m_in_manual_refresh.store(false, std::memory_order_relaxed); ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; success_msg_writer(true) << tr("Refresh done, blocks received: ") << fetched_blocks; - show_balance(); + show_balance_unlocked(); } catch (const tools::error::daemon_busy&) { @@ -1698,8 +1882,6 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset) fail_msg_writer() << tr("refresh failed: ") << ss.str() << ". " << tr("Blocks received: ") << fetched_blocks; } - m_in_manual_refresh.store(false, std::memory_order_relaxed); - m_auto_refresh_run.store(auto_refresh_run, std::memory_order_relaxed); return true; } //---------------------------------------------------------------------------------------------------- @@ -1719,15 +1901,24 @@ bool simple_wallet::refresh(const std::vector<std::string>& args) return refresh_main(start_height, false); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::show_balance(const std::vector<std::string>& args/* = std::vector<std::string>()*/) +bool simple_wallet::show_balance_unlocked() { success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance()) << ", " << tr("unlocked balance: ") << print_money(m_wallet->unlocked_balance()); return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::show_balance(const std::vector<std::string>& args/* = std::vector<std::string>()*/) +{ + LOCK_REFRESH_THREAD_SCOPE(); + show_balance_unlocked(); + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args) { + LOCK_REFRESH_THREAD_SCOPE(); + bool filter = false; bool available = false; if (!args.empty()) @@ -1793,6 +1984,8 @@ bool simple_wallet::show_payments(const std::vector<std::string> &args) return true; } + LOCK_REFRESH_THREAD_SCOPE(); + message_writer() << boost::format("%68s%68s%12s%21s%16s") % tr("payment") % tr("transaction") % tr("height") % tr("amount") % tr("unlock time"); @@ -1870,6 +2063,7 @@ bool simple_wallet::rescan_spent(const std::vector<std::string> &args) try { + LOCK_REFRESH_THREAD_SCOPE(); m_wallet->rescan_spent(); } catch (const tools::error::daemon_busy&) @@ -1903,11 +2097,83 @@ bool simple_wallet::rescan_spent(const std::vector<std::string> &args) return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::get_address_from_str(const std::string &str, cryptonote::account_public_address &address, bool &has_payment_id, crypto::hash8 &payment_id) +{ + if(!get_account_integrated_address_from_str(address, has_payment_id, payment_id, m_wallet->testnet(), str)) + { + // if treating as an address fails, try as url + bool dnssec_ok = false; + std::string url = str; + + // attempt to get address from dns query + auto addresses_from_dns = tools::wallet2::addresses_from_url(url, dnssec_ok); + + // for now, move on only if one address found + if (addresses_from_dns.size() == 1) + { + if (get_account_integrated_address_from_str(address, has_payment_id, payment_id, m_wallet->testnet(), addresses_from_dns[0])) + { + // if it was an address, prompt user for confirmation. + // inform user of DNSSEC validation status as well. + + std::string dnssec_str; + if (dnssec_ok) + { + dnssec_str = tr("DNSSEC validation passed"); + } + else + { + dnssec_str = tr("WARNING: DNSSEC validation was unsuccessful, this address may not be correct!"); + } + std::stringstream prompt; + prompt << tr("For URL: ") << url + << ", " << dnssec_str << std::endl + << tr(" Monero Address = ") << addresses_from_dns[0] + << std::endl + << tr("Is this OK? (Y/n) ") + ; + + // prompt the user for confirmation given the dns query and dnssec status + std::string confirm_dns_ok = command_line::input_line(prompt.str()); + if (std::cin.eof()) + { + return false; + } + if (confirm_dns_ok != "Y" && confirm_dns_ok != "y" && confirm_dns_ok != "Yes" && confirm_dns_ok != "yes" + && confirm_dns_ok != tr("yes") && confirm_dns_ok != tr("no")) + { + fail_msg_writer() << tr("you have cancelled the transfer request"); + return false; + } + } + else + { + fail_msg_writer() << tr("failed to get a Monero address from: ") << url; + return false; + } + } + else if (addresses_from_dns.size() > 1) + { + fail_msg_writer() << tr("not yet supported: Multiple Monero addresses found for given URL: ") << url; + return false; + } + else + { + fail_msg_writer() << tr("wrong address: ") << url; + return false; + } + } + + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::string> &args_) { if (!try_connect_to_daemon()) return true; + LOCK_REFRESH_THREAD_SCOPE(); + std::vector<std::string> local_args = args_; size_t fake_outs_count; @@ -1977,65 +2243,8 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str cryptonote::tx_destination_entry de; bool has_payment_id; crypto::hash8 new_payment_id; - if(!get_account_integrated_address_from_str(de.addr, has_payment_id, new_payment_id, m_wallet->testnet(), local_args[i])) - { - // if treating as an address fails, try as url - bool dnssec_ok = false; - std::string url = local_args[i]; - - // attempt to get address from dns query - auto addresses_from_dns = tools::wallet2::addresses_from_url(url, dnssec_ok); - - // for now, move on only if one address found - if (addresses_from_dns.size() == 1) - { - if (get_account_integrated_address_from_str(de.addr, has_payment_id, new_payment_id, m_wallet->testnet(), addresses_from_dns[0])) - { - // if it was an address, prompt user for confirmation. - // inform user of DNSSEC validation status as well. - - std::string dnssec_str; - if (dnssec_ok) - { - dnssec_str = tr("DNSSEC validation passed"); - } - else - { - dnssec_str = tr("WARNING: DNSSEC validation was unsuccessful, this address may not be correct!"); - } - std::stringstream prompt; - prompt << tr("For URL: ") << url - << ", " << dnssec_str << std::endl - << tr(" Monero Address = ") << addresses_from_dns[0] - << std::endl - << tr("Is this OK? (Y/n) ") - ; - - // prompt the user for confirmation given the dns query and dnssec status - std::string confirm_dns_ok = command_line::input_line(prompt.str()); - if (confirm_dns_ok != "Y" && confirm_dns_ok != "y" && confirm_dns_ok != "Yes" && confirm_dns_ok != "yes" - && confirm_dns_ok != tr("yes") && confirm_dns_ok != tr("no")) - { - fail_msg_writer() << tr("you have cancelled the transfer request"); - return true; - } - } - else - { - fail_msg_writer() << tr("failed to get a Monero address from: ") << local_args[i]; - return true; - } - } - else if (addresses_from_dns.size() > 1) - { - fail_msg_writer() << tr("not yet supported: Multiple Monero addresses found for given URL: ") << url; - } - else - { - fail_msg_writer() << tr("wrong address: ") << local_args[i]; - return true; - } - } + if (!get_address_from_str(local_args[i], de.addr, has_payment_id, new_payment_id)) + return true; if (has_payment_id) { @@ -2071,32 +2280,46 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str // figure out what tx will be necessary std::vector<tools::wallet2::pending_tx> ptx_vector; if (new_algorithm) - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra); + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra, m_trusted_daemon); else - ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra); + ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra, m_trusted_daemon); // if more than one tx necessary, prompt user to confirm if (m_wallet->always_confirm_transfers() || ptx_vector.size() > 1) { uint64_t total_fee = 0; + uint64_t dust_not_in_fee = 0; + uint64_t dust_in_fee = 0; for (size_t n = 0; n < ptx_vector.size(); ++n) { total_fee += ptx_vector[n].fee; + + if (ptx_vector[n].dust_added_to_fee) + dust_in_fee += ptx_vector[n].dust; + else + dust_not_in_fee += ptx_vector[n].dust; } - std::string prompt_str; + std::stringstream prompt; if (ptx_vector.size() > 1) { - prompt_str = (boost::format(tr("Your transaction needs to be split into %llu transactions. " - "This will result in a transaction fee being applied to each transaction, for a total fee of %s. Is this okay? (Y/Yes/N/No)")) % - ((unsigned long long)ptx_vector.size()) % print_money(total_fee)).str(); + prompt << boost::format(tr("Your transaction needs to be split into %llu transactions. " + "This will result in a transaction fee being applied to each transaction, for a total fee of %s")) % + ((unsigned long long)ptx_vector.size()) % print_money(total_fee); } else { - prompt_str = (boost::format(tr("The transaction fee is %s. Is this okay? (Y/Yes/N/No)")) % - print_money(total_fee)).str(); + prompt << boost::format(tr("The transaction fee is %s")) % + print_money(total_fee); } - std::string accepted = command_line::input_line(prompt_str); + if (dust_in_fee != 0) prompt << boost::format(tr(", of which %s is dust from change")) % print_money(dust_in_fee); + if (dust_not_in_fee != 0) prompt << tr(".") << ENDL << boost::format(tr("A total of %s from dust change will be sent to dust address")) + % print_money(dust_not_in_fee); + prompt << tr(".") << ENDL << tr("Is this okay? (Y/Yes/N/No)"); + + std::string accepted = command_line::input_line(prompt.str()); + if (std::cin.eof()) + return true; if (accepted != "Y" && accepted != "y" && accepted != "Yes" && accepted != "yes") { fail_msg_writer() << tr("transaction cancelled."); @@ -2159,6 +2382,9 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str catch (const tools::error::tx_rejected& e) { fail_msg_writer() << (boost::format(tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status(); + std::string reason = e.reason(); + if (!reason.empty()) + fail_msg_writer() << tr("Reason: ") << reason; } catch (const tools::error::tx_sum_overflow& e) { @@ -2218,6 +2444,7 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) return true; } + LOCK_REFRESH_THREAD_SCOPE(); try { // figure out what tx will be necessary @@ -2254,6 +2481,8 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) print_money(total_fee)).str(); } std::string accepted = command_line::input_line(prompt_str); + if (std::cin.eof()) + return true; if (accepted != "Y" && accepted != "y" && accepted != "Yes" && accepted != "yes") { fail_msg_writer() << tr("transaction cancelled."); @@ -2316,6 +2545,241 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) catch (const tools::error::tx_rejected& e) { fail_msg_writer() << (boost::format(tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status(); + std::string reason = e.reason(); + if (!reason.empty()) + fail_msg_writer() << tr("Reason: ") << reason; + } + catch (const tools::error::tx_sum_overflow& e) + { + fail_msg_writer() << e.what(); + } + catch (const tools::error::zero_destination&) + { + fail_msg_writer() << tr("one of destinations is zero"); + } + catch (const tools::error::tx_too_big& e) + { + fail_msg_writer() << tr("failed to find a suitable way to split transactions"); + } + catch (const tools::error::transfer_error& e) + { + LOG_ERROR("unknown transfer error: " << e.to_string()); + fail_msg_writer() << tr("unknown transfer error: ") << e.what(); + } + catch (const tools::error::wallet_internal_error& e) + { + LOG_ERROR("internal error: " << e.to_string()); + fail_msg_writer() << tr("internal error: ") << e.what(); + } + catch (const std::exception& e) + { + LOG_ERROR("unexpected error: " << e.what()); + fail_msg_writer() << tr("unexpected error: ") << e.what(); + } + catch (...) + { + LOG_ERROR("unknown error"); + fail_msg_writer() << tr("unknown error"); + } + + return true; +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::sweep_all(const std::vector<std::string> &args_) +{ + if (!try_connect_to_daemon()) + return true; + + if(m_wallet->watch_only()) + { + fail_msg_writer() << tr("this is a watch only wallet"); + return true; + } + + std::vector<std::string> local_args = args_; + + size_t fake_outs_count; + if(local_args.size() > 0) { + if(!epee::string_tools::get_xtype_from_string(fake_outs_count, local_args[0])) + { + fake_outs_count = m_wallet->default_mixin(); + if (fake_outs_count == 0) + fake_outs_count = DEFAULT_MIX; + } + else + { + local_args.erase(local_args.begin()); + } + } + + std::vector<uint8_t> extra; + bool payment_id_seen = false; + if (2 == local_args.size()) + { + std::string payment_id_str = local_args.back(); + local_args.pop_back(); + + crypto::hash payment_id; + bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); + if(r) + { + std::string extra_nonce; + set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + } + else + { + crypto::hash8 payment_id8; + r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); + if(r) + { + std::string extra_nonce; + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); + r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + } + } + + if(!r) + { + fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str; + return true; + } + payment_id_seen = true; + } + + if (local_args.size() == 0) + { + fail_msg_writer() << tr("No address given"); + return true; + } + + bool has_payment_id; + crypto::hash8 new_payment_id; + cryptonote::account_public_address address; + if (!get_address_from_str(local_args[0], address, has_payment_id, new_payment_id)) + return true; + + if (has_payment_id) + { + if (payment_id_seen) + { + fail_msg_writer() << tr("a single transaction cannot use more than one payment id: ") << local_args[0]; + return true; + } + + std::string extra_nonce; + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, new_payment_id); + bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + if(!r) + { + fail_msg_writer() << tr("failed to set up payment id, though it was decoded correctly"); + return true; + } + } + + try + { + // figure out what tx will be necessary + auto ptx_vector = m_wallet->create_transactions_all(address, fake_outs_count, 0 /* unlock_time */, 0 /* unused fee arg*/, extra, m_trusted_daemon); + + if (ptx_vector.empty()) + { + fail_msg_writer() << tr("No outputs found"); + return true; + } + + // give user total and fee, and prompt to confirm + uint64_t total_fee = 0, total_sent = 0; + for (size_t n = 0; n < ptx_vector.size(); ++n) + { + total_fee += ptx_vector[n].fee; + for (const auto &vin: ptx_vector[n].tx.vin) + { + if (vin.type() == typeid(txin_to_key)) + total_sent += boost::get<txin_to_key>(vin).amount; + } + } + + std::string prompt_str; + if (ptx_vector.size() > 1) { + prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay? (Y/Yes/N/No)")) % + print_money(total_sent) % + ((unsigned long long)ptx_vector.size()) % + print_money(total_fee)).str(); + } + else { + prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No)")) % + print_money(total_sent) % + print_money(total_fee)).str(); + } + std::string accepted = command_line::input_line(prompt_str); + if (std::cin.eof()) + return true; + if (accepted != "Y" && accepted != "y" && accepted != "Yes" && accepted != "yes") + { + fail_msg_writer() << tr("transaction cancelled."); + + // would like to return false, because no tx made, but everything else returns true + // and I don't know what returning false might adversely affect. *sigh* + return true; + } + + // actually commit the transactions + while (!ptx_vector.empty()) + { + auto & ptx = ptx_vector.back(); + m_wallet->commit_tx(ptx); + success_msg_writer(true) << tr("Money successfully sent, transaction: ") << get_transaction_hash(ptx.tx); + + // if no exception, remove element from vector + ptx_vector.pop_back(); + } + } + catch (const tools::error::daemon_busy&) + { + fail_msg_writer() << tr("daemon is busy. Please try again later."); + } + catch (const tools::error::no_connection_to_daemon&) + { + fail_msg_writer() << tr("no connection to daemon. Please make sure daemon is running."); + } + catch (const tools::error::wallet_rpc_error& e) + { + LOG_ERROR("RPC error: " << e.to_string()); + fail_msg_writer() << tr("RPC error: ") << e.what(); + } + catch (const tools::error::get_random_outs_error&) + { + fail_msg_writer() << tr("failed to get random outputs to mix"); + } + catch (const tools::error::not_enough_money& e) + { + fail_msg_writer() << boost::format(tr("not enough money to transfer, available only %s, transaction amount %s = %s + %s (fee).\n%s")) % + print_money(e.available()) % + print_money(e.tx_amount() + e.fee()) % + print_money(e.tx_amount()) % + print_money(e.fee()) % + tr("This is usually due to dust which is so small it cannot pay for itself in fees"); + } + catch (const tools::error::not_enough_outs_to_mix& e) + { + auto writer = fail_msg_writer(); + writer << tr("not enough outputs for specified mixin_count") << " = " << e.mixin_count() << ":"; + for (const cryptonote::COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& outs_for_amount : e.scanty_outs()) + { + writer << "\n" << tr("output amount") << " = " << print_money(outs_for_amount.amount) << ", " << tr("found outputs to mix") << " = " << outs_for_amount.outs.size(); + } + } + catch (const tools::error::tx_not_constructed&) + { + fail_msg_writer() << tr("transaction was not constructed"); + } + catch (const tools::error::tx_rejected& e) + { + fail_msg_writer() << (boost::format(tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status(); + std::string reason = e.reason(); + if (!reason.empty()) + fail_msg_writer() << tr("Reason: ") << reason; } catch (const tools::error::tx_sum_overflow& e) { @@ -2370,6 +2834,8 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) } crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data()); + LOCK_REFRESH_THREAD_SCOPE(); + crypto::secret_key tx_key; bool r = m_wallet->get_tx_key(txid, tx_key); if (r) @@ -2404,6 +2870,8 @@ bool simple_wallet::check_tx_key(const std::vector<std::string> &args_) } crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data()); + LOCK_REFRESH_THREAD_SCOPE(); + cryptonote::blobdata tx_key_data; if(!epee::string_tools::parse_hexstr_to_binbuff(local_args[1], tx_key_data)) { @@ -2425,13 +2893,18 @@ bool simple_wallet::check_tx_key(const std::vector<std::string> &args_) COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); if (!net_utils::invoke_http_json_remote_command2(m_daemon_address + "/gettransactions", req, res, m_http_client) || - res.txs_as_hex.empty()) + (res.txs.empty() && res.txs_as_hex.empty())) { fail_msg_writer() << tr("failed to get transaction from daemon"); return true; } cryptonote::blobdata tx_data; - if (!string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data)) + bool ok; + if (!res.txs.empty()) + ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + else + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + if (!ok) { fail_msg_writer() << tr("failed to parse transaction from daemon"); return true; @@ -2488,6 +2961,23 @@ bool simple_wallet::check_tx_key(const std::vector<std::string> &args_) return true; } //---------------------------------------------------------------------------------------------------- +static std::string get_human_readable_timestamp(uint64_t ts) +{ + char buffer[64]; + if (ts < 1234567890) + return "<unknown>"; + time_t tt = ts; + struct tm tm; + gmtime_r(&tt, &tm); + uint64_t now = time(NULL); + uint64_t diff = ts > now ? ts - now : now - ts; + if (diff > 24*3600) + strftime(buffer, sizeof(buffer), "%F", &tm); + else + strftime(buffer, sizeof(buffer), "%r", &tm); + return std::string(buffer); +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::show_transfers(const std::vector<std::string> &args_) { std::vector<std::string> local_args = args_; @@ -2503,6 +2993,8 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) return true; } + LOCK_REFRESH_THREAD_SCOPE(); + // optional in/out selector if (local_args.size() > 0) { if (local_args[0] == "in" || local_args[0] == "incoming") { @@ -2560,7 +3052,8 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::string payment_id = string_tools::pod_to_hex(i->first); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); - output.insert(std::make_pair(pd.m_block_height, std::make_pair(true, (boost::format("%20.20s %s %s %s") % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % "-").str()))); + std::string note = m_wallet->get_tx_note(pd.m_tx_hash); + output.insert(std::make_pair(pd.m_block_height, std::make_pair(true, (boost::format("%16.16s %20.20s %s %s %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % "-" % note).str()))); } } @@ -2580,7 +3073,8 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); - output.insert(std::make_pair(pd.m_block_height, std::make_pair(false, (boost::format("%20.20s %s %s %14.14s %s") % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests).str()))); + std::string note = m_wallet->get_tx_note(i->first); + output.insert(std::make_pair(pd.m_block_height, std::make_pair(false, (boost::format("%16.16s %20.20s %s %s %14.14s %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % note).str()))); } } @@ -2603,9 +3097,10 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); + std::string note = m_wallet->get_tx_note(i->first); bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed; if ((failed && is_failed) || (!is_failed && pending)) { - message_writer() << (boost::format("%8.8s %6.6s %20.20s %s %s %14.14s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % print_money(amount - pd.m_change) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee)).str(); + message_writer() << (boost::format("%8.8s %6.6s %16.16s %20.20s %s %s %14.14s %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % note).str(); } } } @@ -2657,7 +3152,12 @@ bool simple_wallet::run() void simple_wallet::stop() { m_cmd_binder.stop_handling(); + + m_auto_refresh_run.store(false, std::memory_order_relaxed); m_wallet->stop(); + // make the background refresh thread quit + boost::unique_lock<boost::mutex> lock(m_auto_refresh_mutex); + m_auto_refresh_cond.notify_one(); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::print_address(const std::vector<std::string> &args/* = std::vector<std::string>()*/) @@ -2708,6 +3208,59 @@ bool simple_wallet::print_integrated_address(const std::vector<std::string> &arg return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::set_tx_note(const std::vector<std::string> &args) +{ + if (args.size() == 0) + { + fail_msg_writer() << tr("usage: set_tx_note [txid] free text note"); + return true; + } + + cryptonote::blobdata txid_data; + if(!epee::string_tools::parse_hexstr_to_binbuff(args.front(), txid_data)) + { + fail_msg_writer() << tr("failed to parse txid"); + return false; + } + crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data()); + + std::string note = ""; + for (size_t n = 1; n < args.size(); ++n) + { + if (n > 1) + note += " "; + note += args[n]; + } + m_wallet->set_tx_note(txid, note); + + return true; +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::get_tx_note(const std::vector<std::string> &args) +{ + if (args.size() != 1) + { + fail_msg_writer() << tr("usage: get_tx_note [txid]"); + return true; + } + + cryptonote::blobdata txid_data; + if(!epee::string_tools::parse_hexstr_to_binbuff(args.front(), txid_data)) + { + fail_msg_writer() << tr("failed to parse txid"); + return false; + } + crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data()); + + std::string note = m_wallet->get_tx_note(txid); + if (note.empty()) + success_msg_writer() << "no note found"; + else + success_msg_writer() << "note found: " << note; + + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::process_command(const std::vector<std::string> &args) { return m_cmd_binder.process_command_vec(args); @@ -2735,6 +3288,7 @@ int main(int argc, char* argv[]) std::string lang = i18n_get_language(); tools::sanitize_locale(); + tools::set_strict_default_file_permissions(true); string_tools::set_module_name_and_folder(argv[0]); @@ -2755,6 +3309,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_daemon_port); command_line::add_arg(desc_params, arg_command); command_line::add_arg(desc_params, arg_log_level); + command_line::add_arg(desc_params, arg_max_concurrency); bf::path default_log {log_space::log_singletone::get_default_log_folder()}; std::string log_file_name = log_space::log_singletone::get_default_log_file(); @@ -2782,6 +3337,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_testnet); command_line::add_arg(desc_params, arg_restricted); command_line::add_arg(desc_params, arg_trusted_daemon); + command_line::add_arg(desc_params, arg_restore_height); tools::wallet_rpc_server::init_options(desc_params); po::positional_options_description positional_options; @@ -2839,6 +3395,9 @@ int main(int argc, char* argv[]) LOG_LEVEL_4 ); + if(command_line::has_arg(vm, arg_max_concurrency)) + tools::set_max_concurrency(command_line::get_arg(vm, arg_max_concurrency)); + message_writer(epee::log_space::console_color_white, true) << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")"; if(command_line::has_arg(vm, arg_log_level)) @@ -2891,12 +3450,25 @@ int main(int argc, char* argv[]) } tools::wallet2 wal(testnet,restricted); + bool quit = false; + tools::signal_handler::install([&wal, &quit](int) { + quit = true; + wal.stop(); + }); try { LOG_PRINT_L0(sw::tr("Loading wallet...")); wal.load(wallet_file, password); wal.init(daemon_address); wal.refresh(); + // if we ^C during potentially length load/refresh, there's no server loop yet + if (quit) + { + LOG_PRINT_L0(sw::tr("Storing wallet...")); + wal.store(); + LOG_PRINT_GREEN(sw::tr("Stored ok"), LOG_LEVEL_0); + return 1; + } LOG_PRINT_GREEN(sw::tr("Loaded ok"), LOG_LEVEL_0); } catch (const std::exception& e) @@ -2907,10 +3479,8 @@ int main(int argc, char* argv[]) tools::wallet_rpc_server wrpc(wal); bool r = wrpc.init(vm); CHECK_AND_ASSERT_MES(r, 1, sw::tr("Failed to initialize wallet rpc server")); - tools::signal_handler::install([&wrpc, &wal](int) { wrpc.send_stop_signal(); - wal.store(); }); LOG_PRINT_L0(sw::tr("Starting wallet rpc server")); wrpc.run(); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 21bbfa566..0c69f0440 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -114,6 +114,7 @@ namespace cryptonote bool stop_mining(const std::vector<std::string> &args); bool save_bc(const std::vector<std::string>& args); bool refresh(const std::vector<std::string> &args); + bool show_balance_unlocked(); bool show_balance(const std::vector<std::string> &args = std::vector<std::string>()); bool show_incoming_transfers(const std::vector<std::string> &args); bool show_payments(const std::vector<std::string> &args); @@ -121,6 +122,7 @@ namespace cryptonote bool transfer_main(bool new_algorithm, const std::vector<std::string> &args); bool transfer(const std::vector<std::string> &args); bool transfer_new(const std::vector<std::string> &args); + bool sweep_all(const std::vector<std::string> &args); bool sweep_unmixable(const std::vector<std::string> &args); std::vector<std::vector<cryptonote::tx_destination_entry>> split_amounts( std::vector<cryptonote::tx_destination_entry> dsts, size_t num_splits @@ -137,10 +139,13 @@ namespace cryptonote bool show_transfers(const std::vector<std::string> &args); bool rescan_blockchain(const std::vector<std::string> &args); bool refresh_main(uint64_t start_height, bool reset = false); + bool set_tx_note(const std::vector<std::string> &args); + bool get_tx_note(const std::vector<std::string> &args); uint64_t get_daemon_blockchain_height(std::string& err); bool try_connect_to_daemon(); bool ask_wallet_create_if_needed(); + bool get_address_from_str(const std::string &str, cryptonote::account_public_address &address, bool &has_payment_id, crypto::hash8 &payment_id); /*! * \brief Prints the seed with a nice message @@ -231,6 +236,7 @@ namespace cryptonote bool m_restore_deterministic_wallet; // recover flag bool m_non_deterministic; // old 2-random generation bool m_trusted_daemon; + uint64_t m_restore_height; // optional std::string m_daemon_address; std::string m_daemon_host; diff --git a/src/version.h.in b/src/version.h.in index a0832db91..65b899ed2 100644 --- a/src/version.h.in +++ b/src/version.h.in @@ -1,4 +1,4 @@ #define MONERO_VERSION_TAG "@VERSIONTAG@" -#define MONERO_VERSION "0.9.3.0" +#define MONERO_VERSION "0.9.4.0" #define MONERO_RELEASE_NAME "Hydrogen Helix" #define MONERO_VERSION_FULL MONERO_VERSION "-" MONERO_VERSION_TAG diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 6fd77eead..b6c10c0e5 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -52,6 +52,7 @@ using namespace epee; #include "rapidjson/document.h" #include "rapidjson/writer.h" #include "rapidjson/stringbuffer.h" +#include "common/json_util.h" extern "C" { @@ -72,6 +73,7 @@ using namespace cryptonote; #define KILL_IOSERVICE() \ do { \ work.reset(); \ + while (!ioservice.stopped()) ioservice.poll(); \ threadpool.join_all(); \ ioservice.stop(); \ } while(0) @@ -182,7 +184,7 @@ void wallet2::check_acc_out(const account_keys &acc, const tx_out &o, const cryp error = false; } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_t height, bool miner_tx) +void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_t height, uint64_t ts, bool miner_tx) { if (!miner_tx) process_unconfirmed(tx, height); @@ -211,7 +213,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ tx_pub_key = pub_key_field.pub_key; bool r = true; - int threads = boost::thread::hardware_concurrency(); + int threads = tools::get_max_concurrency(); if (miner_tx && m_refresh_type == RefreshNoCoinbase) { // assume coinbase isn't for us @@ -407,7 +409,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ if (tx_money_spent_in_ins > 0) { - process_outgoing(tx, height, tx_money_spent_in_ins, tx_money_got_in_outs); + process_outgoing(tx, height, ts, tx_money_spent_in_ins, tx_money_got_in_outs); } uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0; @@ -457,6 +459,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ payment.m_amount = received; payment.m_block_height = height; payment.m_unlock_time = tx.unlock_time; + payment.m_timestamp = ts; m_payments.emplace(payment_id, payment); LOG_PRINT_L2("Payment found: " << payment_id << " / " << payment.m_tx_hash << " / " << payment.m_amount); } @@ -480,7 +483,7 @@ void wallet2::process_unconfirmed(const cryptonote::transaction& tx, uint64_t he } } //---------------------------------------------------------------------------------------------------- -void wallet2::process_outgoing(const cryptonote::transaction &tx, uint64_t height, uint64_t spent, uint64_t received) +void wallet2::process_outgoing(const cryptonote::transaction &tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received) { crypto::hash txid = get_transaction_hash(tx); confirmed_transfer_details &ctd = m_confirmed_txs[txid]; @@ -490,6 +493,7 @@ void wallet2::process_outgoing(const cryptonote::transaction &tx, uint64_t heigh ctd.m_amount_out = get_outs_money_amount(tx); ctd.m_change = received; ctd.m_block_height = height; + ctd.m_timestamp = ts; } //---------------------------------------------------------------------------------------------------- void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height) @@ -500,7 +504,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry if(b.timestamp + 60*60*24 > m_account.get_createtime() && height >= m_refresh_from_block_height) { TIME_MEASURE_START(miner_tx_handle_time); - process_new_transaction(b.miner_tx, height, true); + process_new_transaction(b.miner_tx, height, b.timestamp, true); TIME_MEASURE_FINISH(miner_tx_handle_time); TIME_MEASURE_START(txs_handle_time); @@ -509,13 +513,14 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry cryptonote::transaction tx; bool r = parse_and_validate_tx_from_blob(txblob, tx); THROW_WALLET_EXCEPTION_IF(!r, error::tx_parse_error, txblob); - process_new_transaction(tx, height, false); + process_new_transaction(tx, height, b.timestamp, false); } TIME_MEASURE_FINISH(txs_handle_time); LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " << miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms"); }else { - LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime()); + if (!(height % 100)) + LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime()); } m_blockchain.push_back(bl_id); ++m_local_bc_height; @@ -576,12 +581,30 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, blocks = res.blocks; } //---------------------------------------------------------------------------------------------------- +void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes) +{ + cryptonote::COMMAND_RPC_GET_HASHES_FAST::request req = AUTO_VAL_INIT(req); + cryptonote::COMMAND_RPC_GET_HASHES_FAST::response res = AUTO_VAL_INIT(res); + req.block_ids = short_chain_history; + + req.start_height = start_height; + m_daemon_rpc_mutex.lock(); + bool r = net_utils::invoke_http_bin_remote_command2(m_daemon_address + "/gethashes.bin", req, res, m_http_client, WALLET_RCP_CONNECTION_TIMEOUT); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gethashes.bin"); + THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gethashes.bin"); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status); + + blocks_start_height = res.start_height; + hashes = res.m_block_ids; +} +//---------------------------------------------------------------------------------------------------- void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, uint64_t& blocks_added) { size_t current_index = start_height; blocks_added = 0; - int threads = boost::thread::hardware_concurrency(); + int threads = tools::get_max_concurrency(); if (threads > 1) { std::vector<crypto::hash> round_block_hashes(threads); @@ -770,6 +793,62 @@ void wallet2::check_pending_txes() } } //---------------------------------------------------------------------------------------------------- +void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history) +{ + std::list<crypto::hash> hashes; + size_t current_index = m_blockchain.size(); + + while(m_run.load(std::memory_order_relaxed) && current_index < stop_height) + { + pull_hashes(0, blocks_start_height, short_chain_history, hashes); + if (hashes.size() < 3) + return; + if (hashes.size() + current_index < stop_height) { + std::list<crypto::hash>::iterator right; + // drop early 3 off, skipping the genesis block + if (short_chain_history.size() > 3) { + right = short_chain_history.end(); + std::advance(right,-1); + std::list<crypto::hash>::iterator left = right; + std::advance(left, -3); + short_chain_history.erase(left, right); + } + right = hashes.end(); + // prepend 3 more + for (int i = 0; i<3; i++) { + right--; + short_chain_history.push_front(*right); + } + } + current_index = blocks_start_height; + BOOST_FOREACH(auto& bl_id, hashes) + { + if(current_index >= m_blockchain.size()) + { + if (!(current_index % 1000)) + LOG_PRINT_L2( "Skipped block by height: " << current_index); + m_blockchain.push_back(bl_id); + ++m_local_bc_height; + + if (0 != m_callback) + { // FIXME: this isn't right, but simplewallet just logs that we got a block. + cryptonote::block dummy; + m_callback->on_new_block(current_index, dummy); + } + } + else if(bl_id != m_blockchain[current_index]) + { + //split detected here !!! + return; + } + ++current_index; + if (current_index >= stop_height) + return; + } + } +} + +//---------------------------------------------------------------------------------------------------- void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) { received_money = false; @@ -784,9 +863,24 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // pull the first set of blocks get_short_chain_history(short_chain_history); + m_run.store(true, std::memory_order_relaxed); + if (start_height > m_blockchain.size() || m_refresh_from_block_height > m_blockchain.size()) { + if (!start_height) + start_height = m_refresh_from_block_height; + // we can shortcut by only pulling hashes up to the start_height + fast_refresh(start_height, blocks_start_height, short_chain_history); + // regenerate the history now that we've got a full set of hashes + short_chain_history.clear(); + get_short_chain_history(short_chain_history); + start_height = 0; + // and then fall through to regular refresh processing + } + pull_blocks(start_height, blocks_start_height, short_chain_history, blocks); + // always reset start_height to 0 to force short_chain_ history to be used on + // subsequent pulls in this refresh. + start_height = 0; - m_run.store(true, std::memory_order_relaxed); while(m_run.load(std::memory_order_relaxed)) { try @@ -966,6 +1060,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const std::string& p value2.SetInt(m_refresh_type); json.AddMember("refresh_type", value2, json.GetAllocator()); + value2.SetUint64(m_refresh_from_block_height); + json.AddMember("refresh_height", value2, json.GetAllocator()); + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); @@ -1004,7 +1101,7 @@ namespace * \param keys_file_name Name of wallet file * \param password Password of wallet file */ -void wallet2::load_keys(const std::string& keys_file_name, const std::string& password) +bool wallet2::load_keys(const std::string& keys_file_name, const std::string& password) { wallet2::keys_file_data keys_file_data; std::string buf; @@ -1033,35 +1130,48 @@ void wallet2::load_keys(const std::string& keys_file_name, const std::string& pa } else { - account_data = std::string(json["key_data"].GetString(), json["key_data"].GetString() + - json["key_data"].GetStringLength()); - if (json.HasMember("seed_language")) + if (!json.HasMember("key_data")) { - set_seed_language(std::string(json["seed_language"].GetString(), json["seed_language"].GetString() + - json["seed_language"].GetStringLength())); + LOG_ERROR("Field key_data not found in JSON"); + return false; } - if (json.HasMember("watch_only")) + if (!json["key_data"].IsString()) { - m_watch_only = json["watch_only"].GetInt() != 0; + LOG_ERROR("Field key_data found in JSON, but not String"); + return false; } - else + const char *field_key_data = json["key_data"].GetString(); + account_data = std::string(field_key_data, field_key_data + json["key_data"].GetStringLength()); + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, seed_language, std::string, String, false); + if (field_seed_language_found) { - m_watch_only = false; - } - m_always_confirm_transfers = json.HasMember("always_confirm_transfers") && (json["always_confirm_transfers"].GetInt() != 0); - m_store_tx_info = (json.HasMember("store_tx_keys") && (json["store_tx_keys"].GetInt() != 0)) - || (json.HasMember("store_tx_info") && (json["store_tx_info"].GetInt() != 0)); - m_default_mixin = json.HasMember("default_mixin") ? json["default_mixin"].GetUint() : 0; - m_auto_refresh = !json.HasMember("auto_refresh") || (json["auto_refresh"].GetInt() != 0); + set_seed_language(field_seed_language); + } + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, watch_only, int, Int, false); + m_watch_only = field_watch_only_found && field_watch_only; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, always_confirm_transfers, int, Int, false); + m_always_confirm_transfers = field_always_confirm_transfers_found && field_always_confirm_transfers; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, store_tx_keys, int, Int, false); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, store_tx_info, int, Int, false); + m_store_tx_info = (field_store_tx_keys_found && (field_store_tx_keys != 0)) + || (field_store_tx_info_found && (field_store_tx_info != 0)); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, default_mixin, unsigned int, Uint, false); + m_default_mixin = field_default_mixin_found ? field_default_mixin : 0; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, auto_refresh, int, Int, false); + m_auto_refresh = !field_auto_refresh_found || (field_auto_refresh != 0); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, refresh_type, int, Int, false); m_refresh_type = RefreshType::RefreshDefault; - if (json.HasMember("refresh_type")) + if (field_refresh_type_found) { - int type = json["refresh_type"].GetInt(); - if (type == RefreshFull || type == RefreshOptimizeCoinbase || type == RefreshNoCoinbase) - m_refresh_type = (RefreshType)type; + if (field_refresh_type == RefreshFull || field_refresh_type == RefreshOptimizeCoinbase || field_refresh_type == RefreshNoCoinbase) + m_refresh_type = (RefreshType)field_refresh_type; else - LOG_PRINT_L0("Unknown refresh-type value (" << type << "), using default"); + LOG_PRINT_L0("Unknown refresh-type value (" << field_refresh_type << "), using default"); } + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, refresh_height, uint64_t, Uint64, false); + if (field_refresh_height_found) + m_refresh_from_block_height = field_refresh_height; } const cryptonote::account_keys& keys = m_account.get_keys(); @@ -1070,6 +1180,7 @@ void wallet2::load_keys(const std::string& keys_file_name, const std::string& pa if(!m_watch_only) r = r && verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password); + return true; } /*! @@ -1214,7 +1325,7 @@ void wallet2::generate(const std::string& wallet_, const std::string& password, m_account_public_address = account_public_address; m_watch_only = false; - bool r = store_keys(m_keys_file, password, true); + bool r = store_keys(m_keys_file, password, false); THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_testnet)); @@ -1358,7 +1469,10 @@ void wallet2::load(const std::string& wallet_, const std::string& password) bool exists = boost::filesystem::exists(m_keys_file, e); THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file); - load_keys(m_keys_file, password); + if (!load_keys(m_keys_file, password)) + { + THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file); + } LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_testnet)); //keys loaded ok! @@ -1734,47 +1848,12 @@ namespace // returns: // direct return: amount of money found // modified reference: selected_transfers, a list of iterators/indices of input sources -uint64_t wallet2::select_transfers(uint64_t needed_money, bool add_dust, uint64_t dust, bool hf2_rules, std::list<transfer_container::iterator>& selected_transfers) +uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::list<transfer_container::iterator>& selected_transfers, bool trusted_daemon) { - std::vector<size_t> unused_transfers_indices; - std::vector<size_t> unused_dust_indices; - - // aggregate sources available for transfers - // if dust needed, take dust from only one source (so require source has at least dust amount) - for (size_t i = 0; i < m_transfers.size(); ++i) - { - const transfer_details& td = m_transfers[i]; - if (!td.m_spent && is_transfer_unlocked(td)) - { - if (dust < td.amount() && is_valid_decomposed_amount(td.amount())) - unused_transfers_indices.push_back(i); - else - { - // for hf2 rules, we disregard dust, which will be spendable only - // via sweep_dust. If we're asked to add dust, though, we still - // consider them, as this will be a mixin 0 tx (and thus we may - // end up with a tx with one mixable output and N dusty ones). - // This should be made better at some point... - if (!hf2_rules || add_dust) - unused_dust_indices.push_back(i); - } - } - } - - bool select_one_dust = add_dust && !unused_dust_indices.empty(); uint64_t found_money = 0; - while (found_money < needed_money && (!unused_transfers_indices.empty() || !unused_dust_indices.empty())) + while (found_money < needed_money && !unused_transfers_indices.empty()) { - size_t idx; - if (select_one_dust) - { - idx = pop_random_value(unused_dust_indices); - select_one_dust = false; - } - else - { - idx = !unused_transfers_indices.empty() ? pop_random_value(unused_transfers_indices) : pop_random_value(unused_dust_indices); - } + size_t idx = pop_random_value(unused_transfers_indices); transfer_container::iterator it = m_transfers.begin() + idx; selected_transfers.push_back(it); @@ -1793,21 +1872,22 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, const std::v utd.m_dests = dests; utd.m_payment_id = payment_id; utd.m_state = wallet2::unconfirmed_transfer_details::pending; + utd.m_timestamp = time(NULL); } //---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx) +void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon) { - transfer(dsts, fake_outputs_count, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx); + transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx, trusted_daemon); } //---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra) +void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon) { cryptonote::transaction tx; pending_tx ptx; - transfer(dsts, fake_outputs_count, unlock_time, fee, extra, tx, ptx); + transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, tx, ptx, trusted_daemon); } namespace { @@ -1959,13 +2039,14 @@ void wallet2::commit_tx(pending_tx& ptx) COMMAND_RPC_SEND_RAW_TX::request req; req.tx_as_hex = epee::string_tools::buff_to_hex_nodelimer(tx_to_blob(ptx.tx)); + req.do_not_relay = false; COMMAND_RPC_SEND_RAW_TX::response daemon_send_resp; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json_remote_command2(m_daemon_address + "/sendrawtransaction", req, daemon_send_resp, m_http_client, 200000); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "sendrawtransaction"); THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "sendrawtransaction"); - THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, daemon_send_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, daemon_send_resp.status, daemon_send_resp.reason); txid = get_transaction_hash(ptx.tx); crypto::hash payment_id = cryptonote::null_hash; @@ -1984,8 +2065,9 @@ void wallet2::commit_tx(pending_tx& ptx) BOOST_FOREACH(transfer_container::iterator it, ptx.selected_transfers) it->m_spent = true; + //fee includes dust if dust policy specified it. LOG_PRINT_L0("Transaction successfully sent. <" << txid << ">" << ENDL - << "Commission: " << print_money(ptx.fee+ptx.dust) << " (dust: " << print_money(ptx.dust) << ")" << ENDL + << "Commission: " << print_money(ptx.fee) << " (dust sent to dust addr: " << print_money((ptx.dust_added_to_fee ? 0 : ptx.dust)) << ")" << ENDL << "Balance: " << print_money(balance()) << ENDL << "Unlocked: " << print_money(unlocked_balance()) << ENDL << "Please, wait for confirmation for your balance to be unlocked."); @@ -2004,8 +2086,9 @@ void wallet2::commit_tx(std::vector<pending_tx>& ptx_vector) // // this function will make multiple calls to wallet2::transfer if multiple // transactions will be required -std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra) +std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon) { + const std::vector<size_t> unused_transfers_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, trusted_daemon); // failsafe split attempt counter size_t attempt_count = 0; @@ -2035,7 +2118,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto uint64_t needed_fee = 0; do { - transfer(dst_vector, fake_outs_count, unlock_time, needed_fee, extra, tx, ptx); + transfer(dst_vector, fake_outs_count, unused_transfers_indices, unlock_time, needed_fee, extra, tx, ptx, trusted_daemon); auto txBlob = t_serializable_object_to_blob(ptx.tx); needed_fee = calculate_fee(txBlob); } while (ptx.fee < needed_fee); @@ -2242,10 +2325,17 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent return true; }); THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, tx); + + + bool dust_sent_elsewhere = (dust_policy.addr_for_dust.m_view_public_key != change_dts.addr.m_view_public_key + || dust_policy.addr_for_dust.m_spend_public_key != change_dts.addr.m_spend_public_key); + + if (dust_policy.add_to_fee || dust_sent_elsewhere) change_dts.amount -= dust; ptx.key_images = key_images; - ptx.fee = fee; - ptx.dust = dust; + ptx.fee = (dust_policy.add_to_fee ? fee+dust : fee); + ptx.dust = ((dust_policy.add_to_fee || dust_sent_elsewhere) ? dust : 0); + ptx.dust_added_to_fee = dust_policy.add_to_fee; ptx.tx = tx; ptx.change_dts = change_dts; ptx.selected_transfers = selected_transfers; @@ -2268,7 +2358,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent // This system allows for sending (almost) the entire balance, since it does // not generate spurious change in all txes, thus decreasing the instantaneous // usable balance. -std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra) +std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon) { std::vector<size_t> unused_transfers_indices; std::vector<size_t> unused_dust_indices; @@ -2410,7 +2500,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); needed_fee = calculate_fee(txBlob); - available_for_fee = test_ptx.fee + test_ptx.change_dts.amount; + available_for_fee = test_ptx.fee + test_ptx.change_dts.amount + (!test_ptx.dust_added_to_fee ? test_ptx.dust : 0); LOG_PRINT_L2("Made a " << txBlob.size() << " kB tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); @@ -2492,6 +2582,133 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp return ptx_vector; } +std::vector<wallet2::pending_tx> wallet2::create_transactions_all(const cryptonote::account_public_address &address, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon) +{ + std::vector<size_t> unused_transfers_indices; + std::vector<size_t> unused_dust_indices; + uint64_t accumulated_fee, accumulated_outputs, accumulated_change; + struct TX { + std::list<transfer_container::iterator> selected_transfers; + std::vector<cryptonote::tx_destination_entry> dsts; + cryptonote::transaction tx; + pending_tx ptx; + size_t bytes; + }; + std::vector<TX> txes; + uint64_t needed_fee, available_for_fee = 0; + uint64_t upper_transaction_size_limit = get_upper_tranaction_size_limit(); + + // gather all our dust and non dust outputs + for (size_t i = 0; i < m_transfers.size(); ++i) + { + const transfer_details& td = m_transfers[i]; + if (!td.m_spent && is_transfer_unlocked(td)) + { + if (is_valid_decomposed_amount(td.amount())) + unused_transfers_indices.push_back(i); + else + unused_dust_indices.push_back(i); + } + } + LOG_PRINT_L2("Starting with " << unused_transfers_indices.size() << " non-dust outputs and " << unused_dust_indices.size() << " dust outputs"); + + // start with an empty tx + txes.push_back(TX()); + accumulated_fee = 0; + accumulated_outputs = 0; + accumulated_change = 0; + needed_fee = 0; + + // while we have something to send + while (!unused_dust_indices.empty() || !unused_transfers_indices.empty()) { + TX &tx = txes.back(); + + // get a random unspent output and use it to pay next chunk. We try to alternate + // dust and non dust to ensure we never get with only dust, from which we might + // get a tx that can't pay for itself + size_t idx = unused_transfers_indices.empty() ? pop_random_value(unused_dust_indices) : unused_dust_indices.empty() ? pop_random_value(unused_transfers_indices) : ((tx.selected_transfers.size() & 1) || accumulated_outputs > FEE_PER_KB * (upper_transaction_size_limit + 1023) / 1024) ? pop_random_value(unused_dust_indices) : pop_random_value(unused_transfers_indices); + + const transfer_details &td = m_transfers[idx]; + LOG_PRINT_L2("Picking output " << idx << ", amount " << print_money(td.amount())); + + // add this output to the list to spend + tx.selected_transfers.push_back(m_transfers.begin() + idx); + uint64_t available_amount = td.amount(); + accumulated_outputs += available_amount; + + // here, check if we need to sent tx and start a new one + LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " + << upper_transaction_size_limit); + bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || (tx.selected_transfers.size() * (fake_outs_count+1) * APPROXIMATE_INPUT_BYTES >= TX_SIZE_TARGET(upper_transaction_size_limit)); + + if (try_tx) { + cryptonote::transaction test_tx; + pending_tx test_ptx; + + needed_fee = 0; + + tx.dsts.push_back(tx_destination_entry(1, address)); + + LOG_PRINT_L2("Trying to create a tx now, with " << tx.dsts.size() << " destinations and " << + tx.selected_transfers.size() << " outputs"); + transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, unlock_time, needed_fee, extra, + detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); + auto txBlob = t_serializable_object_to_blob(test_ptx.tx); + needed_fee = calculate_fee(txBlob); + available_for_fee = test_ptx.fee + test_ptx.dests[0].amount + test_ptx.change_dts.amount; + LOG_PRINT_L2("Made a " << txBlob.size() << " kB tx, with " << print_money(available_for_fee) << " available for fee (" << + print_money(needed_fee) << " needed)"); + + THROW_WALLET_EXCEPTION_IF(needed_fee > available_for_fee, error::wallet_internal_error, "Transaction cannot pay for itself"); + + do { + LOG_PRINT_L2("We made a tx, adjusting fee and saving it"); + tx.dsts[0].amount = available_for_fee - needed_fee; + transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, unlock_time, needed_fee, extra, + detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); + txBlob = t_serializable_object_to_blob(test_ptx.tx); + needed_fee = calculate_fee(txBlob); + LOG_PRINT_L2("Made an attempt at a final " << ((txBlob.size() + 1023)/1024) << " kB tx, with " << print_money(test_ptx.fee) << + " fee and " << print_money(test_ptx.change_dts.amount) << " change"); + } while (needed_fee > test_ptx.fee); + + LOG_PRINT_L2("Made a final " << ((txBlob.size() + 1023)/1024) << " kB tx, with " << print_money(test_ptx.fee) << + " fee and " << print_money(test_ptx.change_dts.amount) << " change"); + + tx.tx = test_tx; + tx.ptx = test_ptx; + tx.bytes = txBlob.size(); + accumulated_fee += test_ptx.fee; + accumulated_change += test_ptx.change_dts.amount; + if (!unused_transfers_indices.empty() || !unused_dust_indices.empty()) + { + LOG_PRINT_L2("We have more to pay, starting another tx"); + txes.push_back(TX()); + } + } + } + + LOG_PRINT_L1("Done creating " << txes.size() << " transactions, " << print_money(accumulated_fee) << + " total fee, " << print_money(accumulated_change) << " total change"); + + std::vector<wallet2::pending_tx> ptx_vector; + for (std::vector<TX>::iterator i = txes.begin(); i != txes.end(); ++i) + { + TX &tx = *i; + uint64_t tx_money = 0; + for (std::list<transfer_container::iterator>::const_iterator mi = tx.selected_transfers.begin(); mi != tx.selected_transfers.end(); ++mi) + tx_money += (*mi)->amount(); + LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << + ": " << (tx.bytes+1023)/1024 << " kB, sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " outputs to " << tx.dsts.size() << " destination(s), including " << + print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); + ptx_vector.push_back(tx.ptx); + } + + // if we made it this far, we're OK to actually send the transactions + return ptx_vector; +} + uint64_t wallet2::unlocked_dust_balance(const tx_dust_policy &dust_policy) const { uint64_t money = 0; @@ -2688,9 +2905,8 @@ std::vector<uint64_t> wallet2::get_unspent_amounts_vector() return vector; } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon) +std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool trusted_daemon) { - // request all outputs with at least 3 instances, so we can use mixin 2 with epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request> req_t = AUTO_VAL_INIT(req_t); epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); @@ -2699,7 +2915,7 @@ std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_dae req_t.method = "get_output_histogram"; if (trusted_daemon) req_t.params.amounts = get_unspent_amounts_vector(); - req_t.params.min_count = 3; + req_t.params.min_count = count; req_t.params.max_count = 0; bool r = net_utils::invoke_http_json_remote_command2(m_daemon_address + "/json_rpc", req_t, resp_t, m_http_client); m_daemon_rpc_mutex.unlock(); @@ -2713,14 +2929,32 @@ std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_dae mixable.insert(i.amount); } - return select_available_outputs([mixable](const transfer_details &td) { + return select_available_outputs([mixable, atleast](const transfer_details &td) { const uint64_t amount = td.amount(); - if (mixable.find(amount) == mixable.end()) - return true; + if (atleast) { + if (mixable.find(amount) != mixable.end()) + return true; + } + else { + if (mixable.find(amount) == mixable.end()) + return true; + } return false; }); } //---------------------------------------------------------------------------------------------------- +std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon) +{ + // request all outputs with less than 3 instances + return select_available_outputs_from_histogram(3, false, trusted_daemon); +} +//---------------------------------------------------------------------------------------------------- +std::vector<size_t> wallet2::select_available_mixable_outputs(bool trusted_daemon) +{ + // request all outputs with at least 3 instances, so we can use mixin 2 with + return select_available_outputs_from_histogram(3, true, trusted_daemon); +} +//---------------------------------------------------------------------------------------------------- std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bool trusted_daemon) { // From hard fork 1, we don't consider small amounts to be dust anymore @@ -2846,6 +3080,19 @@ std::string wallet2::get_keys_file() const return m_keys_file; } +void wallet2::set_tx_note(const crypto::hash &txid, const std::string ¬e) +{ + m_tx_notes[txid] = note; +} + +std::string wallet2::get_tx_note(const crypto::hash &txid) const +{ + std::unordered_map<crypto::hash, std::string>::const_iterator i = m_tx_notes.find(txid); + if (i == m_tx_notes.end()) + return std::string(); + return i->second; +} + //---------------------------------------------------------------------------------------------------- void wallet2::generate_genesis(cryptonote::block& b) { if (m_testnet) diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index fc700a3de..c2d387acd 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -110,6 +110,7 @@ namespace tools uint64_t m_amount; uint64_t m_block_height; uint64_t m_unlock_time; + uint64_t m_timestamp; }; struct unconfirmed_transfer_details @@ -120,6 +121,7 @@ namespace tools std::vector<cryptonote::tx_destination_entry> m_dests; crypto::hash m_payment_id; enum { pending, pending_not_in_pool, failed } m_state; + uint64_t m_timestamp; }; struct confirmed_transfer_details @@ -130,10 +132,11 @@ namespace tools uint64_t m_block_height; std::vector<cryptonote::tx_destination_entry> m_dests; crypto::hash m_payment_id; + uint64_t m_timestamp; confirmed_transfer_details(): m_amount_in(0), m_amount_out(0), m_change((uint64_t)-1), m_block_height(0), m_payment_id(cryptonote::null_hash) {} confirmed_transfer_details(const unconfirmed_transfer_details &utd, uint64_t height): - m_amount_out(get_outs_money_amount(utd.m_tx)), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id) { get_inputs_money_amount(utd.m_tx, m_amount_in); } + m_amount_out(get_outs_money_amount(utd.m_tx)), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp) { get_inputs_money_amount(utd.m_tx, m_amount_in); } }; typedef std::vector<transfer_details> transfer_container; @@ -143,6 +146,7 @@ namespace tools { cryptonote::transaction tx; uint64_t dust, fee; + bool dust_added_to_fee; cryptonote::tx_destination_entry change_dts; std::list<transfer_container::iterator> selected_transfers; std::string key_images; @@ -274,11 +278,11 @@ namespace tools uint64_t unlocked_balance() const; uint64_t unlocked_dust_balance(const tx_dust_policy &dust_policy) const; template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy); + void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon); template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx); + void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); + void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon); + void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); template<typename T> void transfer_from(const std::vector<size_t> &outs, size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx); template<typename T> @@ -287,8 +291,9 @@ namespace tools void commit_tx(pending_tx& ptx_vector); void commit_tx(std::vector<pending_tx>& ptx_vector); - std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee, const std::vector<uint8_t> extra); - std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra); + std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee, const std::vector<uint8_t> extra, bool trusted_daemon); + std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon); + std::vector<wallet2::pending_tx> create_transactions_all(const cryptonote::account_public_address &address, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra, bool trusted_daemon); std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon); bool check_connection(); void get_transfers(wallet2::transfer_container& incoming_transfers) const; @@ -303,6 +308,7 @@ namespace tools template <class t_archive> inline void serialize(t_archive &a, const unsigned int ver) { + uint64_t dummy_refresh_height = 0; // moved to keys file if(ver < 5) return; a & m_blockchain; @@ -323,7 +329,10 @@ namespace tools a & m_confirmed_txs; if(ver < 11) return; - a & m_refresh_from_block_height; + a & dummy_refresh_height; + if(ver < 12) + return; + a & m_tx_notes; } /*! @@ -364,6 +373,15 @@ namespace tools std::string get_wallet_file() const; std::string get_keys_file() const; + + std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool trusted_daemon); + std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f); + std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon); + std::vector<size_t> select_available_mixable_outputs(bool trusted_daemon); + + void set_tx_note(const crypto::hash &txid, const std::string ¬e); + std::string get_tx_note(const crypto::hash &txid) const; + private: /*! * \brief Stores wallet information to wallet file. @@ -378,8 +396,8 @@ namespace tools * \param keys_file_name Name of wallet file * \param password Password of wallet file */ - void load_keys(const std::string& keys_file_name, const std::string& password); - void process_new_transaction(const cryptonote::transaction& tx, uint64_t height, bool miner_tx); + bool load_keys(const std::string& keys_file_name, const std::string& password); + void process_new_transaction(const cryptonote::transaction& tx, uint64_t height, uint64_t ts, bool miner_tx); void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height); void detach_blockchain(uint64_t height); void get_short_chain_history(std::list<crypto::hash>& ids) const; @@ -387,12 +405,14 @@ namespace tools bool is_transfer_unlocked(const transfer_details& td) const; bool clear(); void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks); + void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes); + void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history); void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, bool &error); void process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, uint64_t& blocks_added); - uint64_t select_transfers(uint64_t needed_money, bool add_dust, uint64_t dust, bool hf2_rules, std::list<transfer_container::iterator>& selected_transfers); + uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::list<transfer_container::iterator>& selected_transfers, bool trusted_daemon); bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const cryptonote::transaction& tx, uint64_t height); - void process_outgoing(const cryptonote::transaction& tx, uint64_t height, uint64_t spent, uint64_t received); + void process_outgoing(const cryptonote::transaction& tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received); void add_unconfirmed_tx(const cryptonote::transaction& tx, const std::vector<cryptonote::tx_destination_entry> &dests, const crypto::hash &payment_id, uint64_t change_amount); void generate_genesis(cryptonote::block& b); void check_genesis(const crypto::hash& genesis_hash) const; //throws @@ -403,8 +423,6 @@ namespace tools uint64_t get_upper_tranaction_size_limit(); void check_pending_txes(); std::vector<uint64_t> get_unspent_amounts_vector(); - std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f); - std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon); cryptonote::account_base m_account; std::string m_daemon_address; @@ -421,6 +439,7 @@ namespace tools payment_container m_payments; std::unordered_map<crypto::key_image, size_t> m_key_images; cryptonote::account_public_address m_account_public_address; + std::unordered_map<crypto::hash, std::string> m_tx_notes; uint64_t m_upper_transaction_size_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value std::atomic<bool> m_run; @@ -441,10 +460,10 @@ namespace tools uint64_t m_refresh_from_block_height; }; } -BOOST_CLASS_VERSION(tools::wallet2, 11) -BOOST_CLASS_VERSION(tools::wallet2::payment_details, 0) -BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 2) -BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 1) +BOOST_CLASS_VERSION(tools::wallet2, 12) +BOOST_CLASS_VERSION(tools::wallet2::payment_details, 1) +BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 3) +BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 2) namespace boost { @@ -474,6 +493,9 @@ namespace boost if (ver < 2) return; a & x.m_state; + if (ver < 3) + return; + a & x.m_timestamp; } template <class Archive> @@ -487,6 +509,9 @@ namespace boost return; a & x.m_dests; a & x.m_payment_id; + if (ver < 2) + return; + a & x.m_timestamp; } template <class Archive> @@ -496,6 +521,9 @@ namespace boost a & x.m_amount; a & x.m_block_height; a & x.m_unlock_time; + if (ver < 1) + return; + a & x.m_timestamp; } template <class Archive> @@ -562,17 +590,17 @@ namespace tools } //---------------------------------------------------------------------------------------------------- template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy) + void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon) { pending_tx ptx; cryptonote::transaction tx; - transfer(dsts, fake_outputs_count, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx); + transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx, trusted_daemon); } template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx) + void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx, bool trusted_daemon) { using namespace cryptonote; // throw if attempting a transaction with no destinations @@ -593,9 +621,7 @@ namespace tools // randomly select inputs for transaction // throw if requested send amount is greater than amount available to send std::list<transfer_container::iterator> selected_transfers; - bool hf2_rules = use_fork_rules(2); // first fork has version 2 - const bool add_dust = (0 == fake_outputs_count) && hf2_rules; - uint64_t found_money = select_transfers(needed_money, add_dust, dust_policy.dust_threshold, hf2_rules, selected_transfers); + uint64_t found_money = select_transfers(needed_money, unused_transfers_indices, selected_transfers, trusted_daemon); THROW_WALLET_EXCEPTION_IF(found_money < needed_money, error::not_enough_money, found_money, needed_money - fee, fee); typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; @@ -711,10 +737,16 @@ namespace tools return true; }); THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, tx); + + bool dust_sent_elsewhere = (dust_policy.addr_for_dust.m_view_public_key != change_dts.addr.m_view_public_key + || dust_policy.addr_for_dust.m_spend_public_key != change_dts.addr.m_spend_public_key); + + if (dust_policy.add_to_fee || dust_sent_elsewhere) change_dts.amount -= dust; ptx.key_images = key_images; - ptx.fee = fee; - ptx.dust = dust; + ptx.fee = (dust_policy.add_to_fee ? fee+dust : fee); + ptx.dust = ((dust_policy.add_to_fee || dust_sent_elsewhere) ? dust : 0); + ptx.dust_added_to_fee = dust_policy.add_to_fee; ptx.tx = tx; ptx.change_dts = change_dts; ptx.selected_transfers = selected_transfers; diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 652b19499..184d8a2a1 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -60,6 +60,7 @@ namespace tools // acc_outs_lookup_error // block_parse_error // get_blocks_error + // get_hashes_error // get_out_indexes_error // tx_parse_error // get_tx_pool_error @@ -107,12 +108,14 @@ namespace tools //---------------------------------------------------------------------------------------------------- const char* const failed_rpc_request_messages[] = { "failed to get blocks", + "failed to get hashes", "failed to get out indices", "failed to get random outs" }; enum failed_rpc_request_message_indices { get_blocks_error_message_index, + get_hashes_error_message_index, get_out_indices_error_message_index, get_random_outs_error_message_index }; @@ -291,6 +294,8 @@ namespace tools //---------------------------------------------------------------------------------------------------- typedef failed_rpc_request<refresh_error, get_blocks_error_message_index> get_blocks_error; //---------------------------------------------------------------------------------------------------- + typedef failed_rpc_request<refresh_error, get_hashes_error_message_index> get_hashes_error; + //---------------------------------------------------------------------------------------------------- typedef failed_rpc_request<refresh_error, get_out_indices_error_message_index> get_out_indices_error; //---------------------------------------------------------------------------------------------------- struct tx_parse_error : public refresh_error @@ -458,15 +463,17 @@ namespace tools //---------------------------------------------------------------------------------------------------- struct tx_rejected : public transfer_error { - explicit tx_rejected(std::string&& loc, const cryptonote::transaction& tx, const std::string& status) + explicit tx_rejected(std::string&& loc, const cryptonote::transaction& tx, const std::string& status, const std::string& reason) : transfer_error(std::move(loc), "transaction was rejected by daemon") , m_tx(tx) , m_status(status) + , m_reason(reason) { } const cryptonote::transaction& tx() const { return m_tx; } const std::string& status() const { return m_status; } + const std::string& reason() const { return m_reason; } std::string to_string() const { @@ -474,12 +481,17 @@ namespace tools ss << transfer_error::to_string() << ", status = " << m_status << ", tx:\n"; cryptonote::transaction tx = m_tx; ss << cryptonote::obj_to_json_str(tx); + if (!m_reason.empty()) + { + ss << " (" << m_reason << ")"; + } return ss.str(); } private: cryptonote::transaction m_tx; std::string m_status; + std::string m_reason; }; //---------------------------------------------------------------------------------------------------- struct tx_sum_overflow : public transfer_error diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index d7d99c2ae..5be9ae5b0 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -232,7 +232,7 @@ namespace tools LOG_PRINT_L1("Requested mixin " << req.mixin << " too low for hard fork 2, using 2"); mixin = 2; } - std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_transactions(dsts, mixin, req.unlock_time, req.fee, extra); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_transactions(dsts, mixin, req.unlock_time, req.fee, extra, req.trusted_daemon); // reject proposed transactions if there are more than one. see on_transfer_split below. if (ptx_vector.size() != 1) @@ -245,9 +245,9 @@ namespace tools m_wallet.commit_tx(ptx_vector); // populate response with tx hash - res.tx_hash = boost::lexical_cast<std::string>(cryptonote::get_transaction_hash(ptx_vector.back().tx)); + res.tx_hash = epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx_vector.back().tx)); if (req.get_tx_key) - res.tx_key = boost::lexical_cast<std::string>(ptx_vector.back().tx_key); + res.tx_key = epee::string_tools::pod_to_hex(ptx_vector.back().tx_key); return true; } catch (const tools::error::daemon_busy& e) @@ -299,18 +299,18 @@ namespace tools } std::vector<wallet2::pending_tx> ptx_vector; if (req.new_algorithm) - ptx_vector = m_wallet.create_transactions_2(dsts, mixin, req.unlock_time, req.fee, extra); + ptx_vector = m_wallet.create_transactions_2(dsts, mixin, req.unlock_time, req.fee, extra, req.trusted_daemon); else - ptx_vector = m_wallet.create_transactions(dsts, mixin, req.unlock_time, req.fee, extra); + ptx_vector = m_wallet.create_transactions(dsts, mixin, req.unlock_time, req.fee, extra, req.trusted_daemon); m_wallet.commit_tx(ptx_vector); // populate response with tx hashes for (auto & ptx : ptx_vector) { - res.tx_hash_list.push_back(boost::lexical_cast<std::string>(cryptonote::get_transaction_hash(ptx.tx))); + res.tx_hash_list.push_back(epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx.tx))); if (req.get_tx_keys) - res.tx_key_list.push_back(boost::lexical_cast<std::string>(ptx.tx_key)); + res.tx_key_list.push_back(epee::string_tools::pod_to_hex(ptx.tx_key)); } return true; @@ -354,9 +354,68 @@ namespace tools // populate response with tx hashes for (auto & ptx : ptx_vector) { - res.tx_hash_list.push_back(boost::lexical_cast<std::string>(cryptonote::get_transaction_hash(ptx.tx))); + res.tx_hash_list.push_back(epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx.tx))); if (req.get_tx_keys) - res.tx_key_list.push_back(boost::lexical_cast<std::string>(ptx.tx_key)); + res.tx_key_list.push_back(epee::string_tools::pod_to_hex(ptx.tx_key)); + } + + return true; + } + catch (const tools::error::daemon_busy& e) + { + er.code = WALLET_RPC_ERROR_CODE_DAEMON_IS_BUSY; + er.message = e.what(); + return false; + } + catch (const std::exception& e) + { + er.code = WALLET_RPC_ERROR_CODE_GENERIC_TRANSFER_ERROR; + er.message = e.what(); + return false; + } + catch (...) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR"; + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er) + { + std::vector<cryptonote::tx_destination_entry> dsts; + std::vector<uint8_t> extra; + + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + + // validate the transfer requested and populate dsts & extra + std::list<wallet_rpc::transfer_destination> destination; + destination.push_back(wallet_rpc::transfer_destination()); + destination.back().amount = 0; + destination.back().address = req.address; + if (!validate_transfer(destination, req.payment_id, dsts, extra, er)) + { + return false; + } + + try + { + std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_transactions_all(dsts[0].addr, req.mixin, req.unlock_time, req.fee, extra, req.trusted_daemon); + + m_wallet.commit_tx(ptx_vector); + + // populate response with tx hashes + for (auto & ptx : ptx_vector) + { + res.tx_hash_list.push_back(epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx.tx))); + if (req.get_tx_keys) + res.tx_key_list.push_back(epee::string_tools::pod_to_hex(ptx.tx_key)); } return true; @@ -434,7 +493,7 @@ namespace tools return false; } res.standard_address = get_account_address_as_str(m_wallet.testnet(),address); - res.payment_id = boost::lexical_cast<std::string>(payment_id); + res.payment_id = epee::string_tools::pod_to_hex(payment_id); return true; } catch (std::exception &e) @@ -627,7 +686,7 @@ namespace tools rpc_transfers.amount = td.amount(); rpc_transfers.spent = td.m_spent; rpc_transfers.global_index = td.m_global_output_index; - rpc_transfers.tx_hash = boost::lexical_cast<std::string>(cryptonote::get_transaction_hash(td.m_tx)); + rpc_transfers.tx_hash = epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(td.m_tx)); rpc_transfers.tx_size = txBlob.size(); res.transfers.push_back(rpc_transfers); } @@ -711,5 +770,164 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er) + { + if (req.txids.size() != req.notes.size()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Different amount of txids and notes"; + return false; + } + + std::list<crypto::hash> txids; + std::list<std::string>::const_iterator i = req.txids.begin(); + while (i != req.txids.end()) + { + cryptonote::blobdata txid_blob; + if(!epee::string_tools::parse_hexstr_to_binbuff(*i++, txid_blob)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_TXID; + er.message = "TX ID has invalid format"; + return false; + } + + crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data()); + txids.push_back(txid); + } + + std::list<crypto::hash>::const_iterator il = txids.begin(); + std::list<std::string>::const_iterator in = req.notes.begin(); + while (il != txids.end()) + { + m_wallet.set_tx_note(*il++, *in++); + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_get_tx_notes(const wallet_rpc::COMMAND_RPC_GET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_GET_TX_NOTES::response& res, epee::json_rpc::error& er) + { + res.notes.clear(); + + std::list<crypto::hash> txids; + std::list<std::string>::const_iterator i = req.txids.begin(); + while (i != req.txids.end()) + { + cryptonote::blobdata txid_blob; + if(!epee::string_tools::parse_hexstr_to_binbuff(*i++, txid_blob)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_TXID; + er.message = "TX ID has invalid format"; + return false; + } + + crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data()); + txids.push_back(txid); + } + + std::list<crypto::hash>::const_iterator il = txids.begin(); + while (il != txids.end()) + { + res.notes.push_back(m_wallet.get_tx_note(*il++)); + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er) + { + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + + uint64_t min_height = 0, max_height = (uint64_t)-1; + if (req.filter_by_height) + { + min_height = req.min_height; + max_height = req.max_height; + } + + if (req.in) + { + std::list<std::pair<crypto::hash, tools::wallet2::payment_details>> payments; + m_wallet.get_payments(payments, min_height, max_height); + for (std::list<std::pair<crypto::hash, tools::wallet2::payment_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) { + res.in.push_back(wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry()); + wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry &entry = res.in.back(); + const tools::wallet2::payment_details &pd = i->second; + + entry.txid = string_tools::pod_to_hex(pd.m_tx_hash); + entry.payment_id = string_tools::pod_to_hex(i->first); + if (entry.payment_id.substr(16).find_first_not_of('0') == std::string::npos) + entry.payment_id = entry.payment_id.substr(0,16); + entry.height = pd.m_block_height; + entry.timestamp = pd.m_timestamp; + entry.amount = pd.m_amount; + entry.fee = 0; // TODO + entry.note = m_wallet.get_tx_note(pd.m_tx_hash); + } + } + + if (req.out) + { + std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>> payments; + m_wallet.get_payments_out(payments, min_height, max_height); + for (std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) { + res.in.push_back(wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry()); + wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry &entry = res.in.back(); + const tools::wallet2::confirmed_transfer_details &pd = i->second; + + entry.txid = string_tools::pod_to_hex(i->first); + entry.payment_id = string_tools::pod_to_hex(i->second.m_payment_id); + if (entry.payment_id.substr(16).find_first_not_of('0') == std::string::npos) + entry.payment_id = entry.payment_id.substr(0,16); + entry.height = pd.m_block_height; + entry.timestamp = pd.m_timestamp; + entry.fee = pd.m_amount_in - pd.m_amount_out; + uint64_t change = pd.m_change == (uint64_t)-1 ? 0 : pd.m_change; // change may not be known + entry.amount = pd.m_amount_in - change - entry.fee; + entry.note = m_wallet.get_tx_note(i->first); + + for (const auto &d: pd.m_dests) { + entry.destinations.push_back(wallet_rpc::transfer_destination()); + wallet_rpc::transfer_destination &td = entry.destinations.back(); + td.amount = d.amount; + td.address = get_account_address_as_str(m_wallet.testnet(), d.addr); + } + } + } + + if (req.pending || req.failed) { + std::list<std::pair<crypto::hash, tools::wallet2::unconfirmed_transfer_details>> upayments; + m_wallet.get_unconfirmed_payments_out(upayments); + for (std::list<std::pair<crypto::hash, tools::wallet2::unconfirmed_transfer_details>>::const_iterator i = upayments.begin(); i != upayments.end(); ++i) { + const tools::wallet2::unconfirmed_transfer_details &pd = i->second; + bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed; + if (!((req.failed && is_failed) || (!is_failed && req.pending))) + continue; + std::list<wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry> &entries = is_failed ? res.failed : res.pending; + entries.push_back(wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry()); + wallet_rpc::COMMAND_RPC_GET_TRANSFERS::entry &entry = entries.back(); + + entry.txid = string_tools::pod_to_hex(i->first); + entry.payment_id = string_tools::pod_to_hex(i->second.m_payment_id); + entry.payment_id = string_tools::pod_to_hex(i->second.m_payment_id); + if (entry.payment_id.substr(16).find_first_not_of('0') == std::string::npos) + entry.payment_id = entry.payment_id.substr(0,16); + entry.height = 0; + entry.timestamp = pd.m_timestamp; + uint64_t amount = 0; + cryptonote::get_inputs_money_amount(pd.m_tx, amount); + entry.fee = amount - get_outs_money_amount(pd.m_tx); + entry.amount = amount - pd.m_change - entry.fee; + entry.note = m_wallet.get_tx_note(i->first); + } + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ } diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 6b41df8fb..8c90aecfe 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -67,6 +67,7 @@ namespace tools MAP_JON_RPC_WE("transfer", on_transfer, wallet_rpc::COMMAND_RPC_TRANSFER) MAP_JON_RPC_WE("transfer_split", on_transfer_split, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT) MAP_JON_RPC_WE("sweep_dust", on_sweep_dust, wallet_rpc::COMMAND_RPC_SWEEP_DUST) + MAP_JON_RPC_WE("sweep_all", on_sweep_all, wallet_rpc::COMMAND_RPC_SWEEP_ALL) MAP_JON_RPC_WE("store", on_store, wallet_rpc::COMMAND_RPC_STORE) MAP_JON_RPC_WE("get_payments", on_get_payments, wallet_rpc::COMMAND_RPC_GET_PAYMENTS) MAP_JON_RPC_WE("get_bulk_payments", on_get_bulk_payments, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS) @@ -76,6 +77,9 @@ namespace tools MAP_JON_RPC_WE("split_integrated_address", on_split_integrated_address, wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS) MAP_JON_RPC_WE("stop_wallet", on_stop_wallet, wallet_rpc::COMMAND_RPC_STOP_WALLET) MAP_JON_RPC_WE("rescan_blockchain", on_rescan_blockchain, wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN) + MAP_JON_RPC_WE("set_tx_notes", on_set_tx_notes, wallet_rpc::COMMAND_RPC_SET_TX_NOTES) + MAP_JON_RPC_WE("get_tx_notes", on_get_tx_notes, wallet_rpc::COMMAND_RPC_GET_TX_NOTES) + MAP_JON_RPC_WE("get_transfers", on_get_transfers, wallet_rpc::COMMAND_RPC_GET_TRANSFERS) END_JSON_RPC_MAP() END_URI_MAP2() @@ -87,6 +91,7 @@ namespace tools bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er); bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er); bool on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er); + bool on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er); bool on_make_integrated_address(const wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er); bool on_split_integrated_address(const wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er); bool on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er); @@ -95,6 +100,9 @@ namespace tools bool on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er); bool on_stop_wallet(const wallet_rpc::COMMAND_RPC_STOP_WALLET::request& req, wallet_rpc::COMMAND_RPC_STOP_WALLET::response& res, epee::json_rpc::error& er); bool on_rescan_blockchain(const wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::request& req, wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::response& res, epee::json_rpc::error& er); + bool on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er); + bool on_get_tx_notes(const wallet_rpc::COMMAND_RPC_GET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_GET_TX_NOTES::response& res, epee::json_rpc::error& er); + bool on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er); bool handle_command_line(const boost::program_options::variables_map& vm); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 2c4e26406..f8c04c007 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -115,6 +115,7 @@ namespace wallet_rpc uint64_t unlock_time; std::string payment_id; bool get_tx_key; + bool trusted_daemon; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(destinations) @@ -123,6 +124,7 @@ namespace wallet_rpc KV_SERIALIZE(unlock_time) KV_SERIALIZE(payment_id) KV_SERIALIZE(get_tx_key) + KV_SERIALIZE(trusted_daemon) END_KV_SERIALIZE_MAP() }; @@ -149,6 +151,7 @@ namespace wallet_rpc std::string payment_id; bool new_algorithm; bool get_tx_keys; + bool trusted_daemon; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(destinations) @@ -158,6 +161,7 @@ namespace wallet_rpc KV_SERIALIZE(payment_id) KV_SERIALIZE(new_algorithm) KV_SERIALIZE(get_tx_keys) + KV_SERIALIZE(trusted_daemon) END_KV_SERIALIZE_MAP() }; @@ -198,6 +202,41 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_SWEEP_ALL + { + struct request + { + std::string address; + uint64_t fee; + uint64_t mixin; + uint64_t unlock_time; + std::string payment_id; + bool get_tx_keys; + bool trusted_daemon; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(fee) + KV_SERIALIZE(mixin) + KV_SERIALIZE(unlock_time) + KV_SERIALIZE(payment_id) + KV_SERIALIZE(get_tx_keys) + KV_SERIALIZE(trusted_daemon) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::list<std::string> tx_hash_list; + std::list<std::string> tx_key_list; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(tx_hash_list) + KV_SERIALIZE(tx_key_list) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_STORE { struct request @@ -408,6 +447,110 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_SET_TX_NOTES + { + struct request + { + std::list<std::string> txids; + std::list<std::string> notes; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(txids) + KV_SERIALIZE(notes) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_GET_TX_NOTES + { + struct request + { + std::list<std::string> txids; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(txids) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::list<std::string> notes; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(notes) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_GET_TRANSFERS + { + struct request + { + bool in; + bool out; + bool pending; + bool failed; + + bool filter_by_height; + uint64_t min_height; + uint64_t max_height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(in); + KV_SERIALIZE(out); + KV_SERIALIZE(pending); + KV_SERIALIZE(failed); + KV_SERIALIZE(filter_by_height); + KV_SERIALIZE(min_height); + KV_SERIALIZE(max_height); + END_KV_SERIALIZE_MAP() + }; + + struct entry + { + std::string txid; + std::string payment_id; + uint64_t height; + uint64_t timestamp; + uint64_t amount; + uint64_t fee; + std::string note; + std::list<transfer_destination> destinations; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(txid); + KV_SERIALIZE(payment_id); + KV_SERIALIZE(height); + KV_SERIALIZE(timestamp); + KV_SERIALIZE(amount); + KV_SERIALIZE(fee); + KV_SERIALIZE(note); + KV_SERIALIZE(destinations); + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::list<entry> in; + std::list<entry> out; + std::list<entry> pending; + std::list<entry> failed; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(in); + KV_SERIALIZE(out); + KV_SERIALIZE(pending); + KV_SERIALIZE(failed); + END_KV_SERIALIZE_MAP() + }; + }; + } } diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index 063421d80..968836671 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -38,3 +38,4 @@ #define WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID -5 #define WALLET_RPC_ERROR_CODE_TRANSFER_TYPE -6 #define WALLET_RPC_ERROR_CODE_DENIED -7 +#define WALLET_RPC_ERROR_CODE_WRONG_TXID -8 |