From 132c666f67937676bea1cc145c242b857f9852a2 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 31 Jan 2016 05:10:14 -0800 Subject: Update schema for "tx_outputs" to use array containing amount output indices This speeds up wallet refresh by directly retrieving a tx's amount output indices. It removes the indirection and walking the amount output duplicate list for every amount in each requested tx. "tx_outputs" is used by: Amount output indices are needed for wallet refresh. Global output indices are needed for removing a tx. Both amount output indices and global output indices are now stored in an array of 64-bit unsigned ints: tx_outputs[] -> [ ] Previously it was: tx_outputs[] -> duplicate list of The amount output list had to be walked for every amount in order to find each amount's output index, by comparing the amount's global output index with each one in the duplicate list until a match was found. See also d045dfa7ce0bf131681193c97560da26f9f37900 --- src/blockchain_db/blockchain_db.h | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'src/blockchain_db/blockchain_db.h') diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 3396b8c20..2cdaaea6f 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -297,7 +297,19 @@ private: virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; // tells the subclass to store an output - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0; + virtual void add_output(const crypto::hash& tx_hash, + const tx_out& tx_output, + const uint64_t& local_index, + const uint64_t unlock_time, + uint64_t& amount_output_index, + uint64_t& global_output_index + ) = 0; + + // tells the subclass to store indices for a tx's outputs, both amount output indices and global output indices + virtual void add_amount_and_global_output_indices(const crypto::hash& tx_hash, + const std::vector& amount_output_indices, + const std::vector& global_output_indices + ) = 0; // tells the subclass to remove an output virtual void remove_output(const tx_out& tx_output) = 0; @@ -501,9 +513,13 @@ public: virtual bool can_thread_bulk_indices() const = 0; - // return a vector of indices corresponding to the global output index for - // each output in the transaction with hash - virtual std::vector get_tx_output_indices(const crypto::hash& h) const = 0; + // return two vectors of indices: vector of amount output indices and global + // output indices, corresponding to each output in the transaction with hash + // + virtual void get_amount_and_global_output_indices(const crypto::hash& h, + std::vector& amount_output_indices, + std::vector& global_output_indices) const = 0; + // return a vector of indices corresponding to the amount output index for // each output in the transaction with hash virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const = 0; -- cgit v1.2.3 From 8d12a8df2cf27713509c039c91fb9180e010011a Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 10:59:20 -0800 Subject: Schema update: tx_indices - improve further with less indirection --- src/blockchain_db/blockchain_db.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/blockchain_db/blockchain_db.h') diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 2cdaaea6f..933cc34f5 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -473,6 +473,7 @@ public: // return true if a transaction with hash exists virtual bool tx_exists(const crypto::hash& h) const = 0; + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const = 0; // return unlock time of tx with hash virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const = 0; @@ -516,13 +517,13 @@ public: // return two vectors of indices: vector of amount output indices and global // output indices, corresponding to each output in the transaction with hash // - virtual void get_amount_and_global_output_indices(const crypto::hash& h, + virtual void get_amount_and_global_output_indices(const uint64_t tx_index, std::vector& amount_output_indices, std::vector& global_output_indices) const = 0; // return a vector of indices corresponding to the amount output index for // each output in the transaction with hash - virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const = 0; + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const = 0; // returns true if key image is present in spent key images storage virtual bool has_key_image(const crypto::key_image& img) const = 0; -- cgit v1.2.3 From a2f518aa014163e63e39ce5e83a4f0d4d5be5c3f Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 11:37:41 -0800 Subject: Schema update: tx_indices - yet less indirection --- src/blockchain_db/blockchain_db.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/blockchain_db/blockchain_db.h') diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 933cc34f5..b71bb4416 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -291,7 +291,7 @@ private: virtual void remove_block() = 0; // tells the subclass to store the transaction and its metadata - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; // tells the subclass to remove data about a transaction virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; @@ -306,7 +306,7 @@ private: ) = 0; // tells the subclass to store indices for a tx's outputs, both amount output indices and global output indices - virtual void add_amount_and_global_output_indices(const crypto::hash& tx_hash, + virtual void add_amount_and_global_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices, const std::vector& global_output_indices ) = 0; -- cgit v1.2.3 From 9aadedb1d0eb4f922b33edfa0ede1f4d1ac128d1 Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 11:56:36 -0800 Subject: Schema update: tx_indices - consolidate the tx subdbs from 5 to 3 --- src/blockchain_db/blockchain_db.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'src/blockchain_db/blockchain_db.h') diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index b71bb4416..665cccd40 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -148,6 +148,15 @@ struct output_data_t }; #pragma pack(pop) +#pragma pack(push, 1) +struct tx_data_t +{ + uint64_t tx_index; + uint64_t unlock_time; + uint64_t height; +}; +#pragma pack(pop) + /*********************************** * Exception Definitions ***********************************/ -- cgit v1.2.3 From 591e421875988ea63313a7da22062f62f06b30ab Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 4 Apr 2016 02:10:58 +0100 Subject: Cleanup and clarify Try to rationalize the variable names, document usage. --- src/blockchain_db/blockchain_db.h | 58 +++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 24 deletions(-) (limited to 'src/blockchain_db/blockchain_db.h') diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 665cccd40..99b520d8b 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -63,6 +63,26 @@ * of this interface call to private members of the interface to be added * later that will then populate as needed. * + * Indices and Identifiers: + * The word "index" is used ambiguously throughout this code. It is + * particularly confusing when talking about the output or transaction + * tables since their indexing can refer to themselves or each other. + * I have attempted to clarify these usages here: + * + * Blocks, transactions, and outputs are all identified by a hash. + * For storage efficiency, a 64-bit integer ID is used instead of the hash + * inside the DB. Tables exist to map between hash and ID. A block ID is + * also referred to as its "height". Transactions and outputs generally are + * not referred to by ID outside of this module, but the tx ID is returned + * by tx_exists() and used by get_tx_amount_output_indices(). Like their + * corresponding hashes, IDs are globally unique. + * + * The remaining uses of the word "index" refer to local offsets, and are + * not globally unique. An "amount output index" N refers to the Nth output + * of a specific amount. An "output local index" N refers to the Nth output + * of a specific tx. + * + * * General: * open() * is_open() @@ -99,7 +119,7 @@ * void pop_block(block&, tx_list&) * * Transactions: - * bool tx_exists(hash) + * bool tx_exists(hash, tx_id) * uint64_t get_tx_unlock_time(hash) * tx get_tx(hash) * uint64_t get_tx_count() @@ -111,7 +131,7 @@ * pub_key get_output_key(amount, index) * hash,index get_output_tx_and_index_from_global(index) * hash,index get_output_tx_and_index(amount, index) - * vec get_tx_output_indices(tx_hash) + * vec get_tx_amount_output_indices(tx_id) * * * Spent Output Key Images: @@ -151,9 +171,9 @@ struct output_data_t #pragma pack(push, 1) struct tx_data_t { - uint64_t tx_index; + uint64_t tx_id; uint64_t unlock_time; - uint64_t height; + uint64_t block_id; }; #pragma pack(pop) @@ -299,25 +319,22 @@ private: // tells the subclass to remove data about the top block virtual void remove_block() = 0; - // tells the subclass to store the transaction and its metadata + // tells the subclass to store the transaction and its metadata, returns tx ID virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; // tells the subclass to remove data about a transaction virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; - // tells the subclass to store an output - virtual void add_output(const crypto::hash& tx_hash, + // tells the subclass to store an output, returns amount output index + virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, - const uint64_t unlock_time, - uint64_t& amount_output_index, - uint64_t& global_output_index + const uint64_t unlock_time ) = 0; - // tells the subclass to store indices for a tx's outputs, both amount output indices and global output indices - virtual void add_amount_and_global_output_indices(const uint64_t tx_index, - const std::vector& amount_output_indices, - const std::vector& global_output_indices + // tells the subclass to store amount output indices for a tx's outputs + virtual void add_tx_amount_output_indices(const uint64_t tx_id, + const std::vector& amount_output_indices ) = 0; // tells the subclass to remove an output @@ -482,7 +499,7 @@ public: // return true if a transaction with hash exists virtual bool tx_exists(const crypto::hash& h) const = 0; - virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const = 0; + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_id) const = 0; // return unlock time of tx with hash virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const = 0; @@ -523,16 +540,9 @@ public: virtual bool can_thread_bulk_indices() const = 0; - // return two vectors of indices: vector of amount output indices and global - // output indices, corresponding to each output in the transaction with hash - // - virtual void get_amount_and_global_output_indices(const uint64_t tx_index, - std::vector& amount_output_indices, - std::vector& global_output_indices) const = 0; - // return a vector of indices corresponding to the amount output index for - // each output in the transaction with hash - virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const = 0; + // each output in the transaction with ID + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_id) const = 0; // returns true if key image is present in spent key images storage virtual bool has_key_image(const crypto::key_image& img) const = 0; -- cgit v1.2.3 From 372acee72303b4d7ea225981c523d328fda297be Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 4 Apr 2016 17:28:31 +0100 Subject: Cleanup drop obsolete remove_output() fix get_output_key(global), fix crash in blockchain_dump --- src/blockchain_db/blockchain_db.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/blockchain_db/blockchain_db.h') diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 99b520d8b..93527b484 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -337,9 +337,6 @@ private: const std::vector& amount_output_indices ) = 0; - // tells the subclass to remove an output - virtual void remove_output(const tx_out& tx_output) = 0; - // tells the subclass to store a spent key virtual void add_spent_key(const crypto::key_image& k_image) = 0; -- cgit v1.2.3