diff options
Diffstat (limited to 'src/blockchain_db/lmdb/db_lmdb.cpp')
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.cpp | 664 |
1 files changed, 94 insertions, 570 deletions
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 3ab2b7154..8c51c09b1 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1142,8 +1142,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 // We don't handle the old format previous to that commit. txn.commit(); - migrate(*(const uint32_t *)v.mv_data); m_open = true; + migrate(*(const uint32_t *)v.mv_data); return; } #endif @@ -2777,13 +2777,6 @@ void BlockchainLMDB::fixup() BlockchainDB::fixup(); } -static int compare_amtidx(const MDB_val *a, const MDB_val *b) -{ - const uint64_t *pa = (const uint64_t *)a->mv_data; - const uint64_t *pb = (const uint64_t *)b->mv_data; - return (pa[0] < pb[1]) ? -1 : pa[0] > pb[1]; -} - #define RENAME_DB(name) \ k.mv_data = (void *)name; \ k.mv_size = sizeof(name)-1; \ @@ -3125,283 +3118,107 @@ void BlockchainLMDB::migrate_0_1() txn.commit(); } while(0); - LOG_PRINT_L0("Total number of outputs: " << m_num_outputs); - LOG_PRINT_L1("outputs migration will update output_amounts and output_txs..."); - do { - LOG_PRINT_L1("migrating output_amounts:"); + LOG_PRINT_L1("deleting old indices:"); - MDB_dbi keys; + /* Delete all other tables, we're just going to recreate them */ + MDB_dbi dbi; result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_dbi_open(txn, "output_keys", 0, &keys); - if (result == MDB_NOTFOUND) { - txn.abort(); - LOG_PRINT_L1(" output_amounts already migrated"); - break; - } - MDB_dbi o_amts; - outkey ok; - MDB_val_set(nv, ok); - /* the output_amounts table name is the same but the old version and new version - * have incompatible DB flags. Create a new table with the right flags. - */ - o_amts = m_output_amounts; - lmdb_db_open(txn, "output_amountr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_amounts, "Failed to open db handle for output_amountr"); - mdb_set_dupsort(txn, m_output_amounts, compare_uint64); - - MDB_cursor *c_oamts, *c_keys, *c_cur; - i = 0; - z = m_num_outputs; - uint64_t j; - mdb_size_t num_elems; - MDB_val v2; - - while(1) { - if (!(i % 1000)) { - if (i) { - LOGIF(1) { - std::cout << i << " / " << z << " \r" << std::flush; - } - txn.commit(); - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - } - result = mdb_cursor_open(txn, m_output_amounts, &c_cur); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amountr: ", result).c_str())); - result = mdb_cursor_open(txn, o_amts, &c_oamts); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amounts: ", result).c_str())); - result = mdb_cursor_open(txn, keys, &c_keys); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_keys: ", result).c_str())); - if (!i) { - MDB_stat ms; - mdb_stat(txn, m_output_amounts, &ms); - i = ms.ms_entries; - } - } - result = mdb_cursor_get(c_oamts, &k, &v, MDB_FIRST); - if (result == MDB_NOTFOUND) { - txn.commit(); - break; - } else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from output_amounts: ", result).c_str())); - result = mdb_cursor_count(c_oamts, &num_elems); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get number of outputs for amount: ", result).c_str())); - for (j=0; j<num_elems; j++,i++) { - if (!(i % 1000)) { - uint64_t amt = *(uint64_t *)k.mv_data; - uint64_t oid = *(uint64_t *)v.mv_data; - LOGIF(1) { - std::cout << i << " / " << z << " \r" << std::flush; - } - txn.commit(); - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_cursor_open(txn, m_output_amounts, &c_cur); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amountr: ", result).c_str())); - result = mdb_cursor_open(txn, o_amts, &c_oamts); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amounts: ", result).c_str())); - result = mdb_cursor_open(txn, keys, &c_keys); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_keys: ", result).c_str())); - k.mv_data = (void *)&amt; - v.mv_data = (void *)&oid; - result = mdb_cursor_get(c_oamts, &k, &v, MDB_GET_BOTH); - /* should never fail since we already had this record before the commit */ - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from output_amounts: ", result).c_str())); - } - result = mdb_cursor_get(c_keys, &v, &v2, MDB_SET); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get output_key for amount: ", result).c_str())); - ok.amount_index = j; - ok.output_id = *(uint64_t *)v.mv_data; - ok.data = *(output_data_t *)v2.mv_data; - result = mdb_cursor_put(c_cur, &k, &nv, MDB_APPENDDUP); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into output_amountr: ", result).c_str())); - result = mdb_cursor_del(c_keys, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from output_keys: ", result).c_str())); - result = mdb_cursor_get(c_oamts, &k, &v, MDB_NEXT_DUP); - if (result == MDB_NOTFOUND) - break; - else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from output_amounts: ", result).c_str())); - } - result = mdb_cursor_del(c_oamts, MDB_NODUPDATA); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from output_amounts: ", result).c_str())); - } - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_drop(txn, keys, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete output_keys from the db: ", result).c_str())); - result = mdb_drop(txn, o_amts, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete output_amounts from the db: ", result).c_str())); - - RENAME_DB("output_amountr"); - - /* close and reopen to get old dbi slot back */ - mdb_dbi_close(m_env, m_output_amounts); - lmdb_db_open(txn, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, m_output_amounts, "Failed to open db handle for output_amounts"); - mdb_set_dupsort(txn, m_output_amounts, compare_uint64); - txn.commit(); - } while(0); - - do { - LOG_PRINT_L1("migrating output_txs:"); - - MDB_dbi indices; - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_dbi_open(txn, "output_indices", 0, &indices); + result = mdb_dbi_open(txn, "tx_unlocks", 0, &dbi); if (result == MDB_NOTFOUND) { - txn.abort(); - LOG_PRINT_L1(" output_txs already migrated"); - break; - } - MDB_dbi o_otxs; - outtx ot; - MDB_val_set(nv, ot); - - /* the output_txs table name is the same but the old version and new version - * have incompatible DB flags. Create a new table with the right flags. - */ - o_otxs = m_output_txs; - lmdb_db_open(txn, "output_txr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for output_txr"); - mdb_set_dupsort(txn, m_output_txs, compare_uint64); - - - MDB_cursor *c_otxs, *c_oixs, *c_cur; - i = 0; - z = m_num_outputs; - - while(1) { - if (!(i % 2000)) { - if (i) { - LOGIF(1) { - std::cout << i << " / " << z << " \r" << std::flush; - } - txn.commit(); - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - } - result = mdb_cursor_open(txn, m_output_txs, &c_cur); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_txr: ", result).c_str())); - result = mdb_cursor_open(txn, o_otxs, &c_otxs); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_txs: ", result).c_str())); - result = mdb_cursor_open(txn, indices, &c_oixs); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_indices: ", result).c_str())); - if (!i) { - MDB_stat ms; - mdb_stat(txn, m_output_txs, &ms); - i = ms.ms_entries; - } - } - result = mdb_cursor_get(c_otxs, &k, &v, MDB_NEXT); - if (result == MDB_NOTFOUND) { - txn.commit(); + txn.abort(); + LOG_PRINT_L1(" old indices already deleted"); break; - } else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from output_txs: ", result).c_str())); - ot.output_id = *(uint64_t *)k.mv_data; - ot.tx_hash = *(crypto::hash *)v.mv_data; - result = mdb_cursor_get(c_oixs, &k, &v, MDB_NEXT); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from output_indices: ", result).c_str())); - ot.local_index = *(uint64_t *)v.mv_data; - result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into output_txr: ", result).c_str())); - result = mdb_cursor_del(c_otxs, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from output_txs: ", result).c_str())); - result = mdb_cursor_del(c_oixs, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from output_indices: ", result).c_str())); - i++; } + txn.abort(); + +#define DELETE_DB(x) do { \ + LOG_PRINT_L1(" " x ":"); \ + result = mdb_txn_begin(m_env, NULL, 0, txn); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); \ + result = mdb_dbi_open(txn, x, 0, &dbi); \ + if (!result) { \ + result = mdb_drop(txn, dbi, 1); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to delete " x ": ", result).c_str())); \ + txn.commit(); \ + } } while(0) + + DELETE_DB("tx_heights"); + DELETE_DB("output_txs"); + DELETE_DB("output_indices"); + DELETE_DB("output_keys"); + DELETE_DB("spent_keys"); + DELETE_DB("output_amounts"); + DELETE_DB("tx_outputs"); + DELETE_DB("tx_unlocks"); + + /* reopen new DBs with correct flags */ result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_drop(txn, indices, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete output_indices from the db: ", result).c_str())); - result = mdb_drop(txn, o_otxs, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete output_txs from the db: ", result).c_str())); - - RENAME_DB("output_txr"); - mdb_dbi_close(m_env, m_output_txs); - lmdb_db_open(txn, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for output_txs"); + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_txs, "Failed to open db handle for m_output_txs"); mdb_set_dupsort(txn, m_output_txs, compare_uint64); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys"); + mdb_set_dupsort(txn, m_spent_keys, compare_hash32); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); txn.commit(); + m_num_txs = 0; + m_num_outputs = 0; } while(0); - LOG_PRINT_L0("Total number of txs: " << m_num_txs); - LOG_PRINT_L1("txs migration will update tx_indices, tx_outputs, and txs..."); - do { - LOG_PRINT_L1("migrating tx_indices:"); + LOG_PRINT_L1("migrating txs and outputs:"); - /* The existing indices lack information that version 1 needs, so we just regenerate it - * all from the original Blocks. - */ - MDB_dbi o_tx_unlocks; + unsigned int flags; result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_dbi_open(txn, "tx_unlocks", 0, &o_tx_unlocks); - if (result == MDB_NOTFOUND) { + result = mdb_dbi_flags(txn, m_txs, &flags); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve txs flags: ", result).c_str())); + /* if the flags are what we expect, this table has already been migrated */ + if (flags & MDB_INTEGERKEY) { txn.abort(); - LOG_PRINT_L1(" tx_indices already migrated"); + LOG_PRINT_L1(" txs already migrated"); break; } - MDB_dbi o_tx_heights; - MDB_cursor *c_cur, *c_blocks, *c_props; - MDB_cursor *c_tx_unlocks, *c_tx_heights; - - - lmdb_db_open(txn, "tx_heights", 0, o_tx_heights, "Failed to open db handle for tx_heights"); - txindex ti; - MDB_val_set(iv, ti); - unsigned int j; + MDB_dbi o_txs; blobdata bd; block b; - uint64_t num_txs = m_num_txs; + MDB_val hk; + o_txs = m_txs; + mdb_set_compare(txn, o_txs, compare_hash32); + lmdb_db_open(txn, "txr", MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for txr"); + + txn.commit(); + + MDB_cursor *c_blocks, *c_txs, *c_props, *c_cur; i = 0; - ti.data.block_id = 0; - z = m_num_txs; + z = m_height; + + hk.mv_size = sizeof(crypto::hash); + set_batch_transactions(true); + batch_start(1000); + txn.m_txn = m_write_txn->m_txn; + m_height = 0; while(1) { - if (!(ti.data.block_id % 1000)) { + if (!(i % 1000)) { if (i) { LOGIF(1) { std::cout << i << " / " << z << " \r" << std::flush; } MDB_val_set(pk, "txblk"); - MDB_val_set(pv, ti.data.block_id); + MDB_val_set(pv, m_height); result = mdb_cursor_put(c_props, &pk, &pv, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to update txblk property: ", result).c_str())); @@ -3409,46 +3226,35 @@ void BlockchainLMDB::migrate_0_1() result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + m_write_txn->m_txn = txn.m_txn; + m_write_batch_txn->m_txn = txn.m_txn; + memset(&m_wcursors, 0, sizeof(m_wcursors)); } - result = mdb_cursor_open(txn, m_tx_indices, &c_cur); + result = mdb_cursor_open(txn, m_blocks, &c_blocks); if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); result = mdb_cursor_open(txn, m_properties, &c_props); if (result) throw0(DB_ERROR(lmdb_error("Failed to open a cursor for properties: ", result).c_str())); - result = mdb_cursor_open(txn, o_tx_unlocks, &c_tx_unlocks); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_unlocks: ", result).c_str())); - result = mdb_cursor_open(txn, o_tx_heights, &c_tx_heights); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_heights: ", result).c_str())); - result = mdb_cursor_open(txn, m_blocks, &c_blocks); + result = mdb_cursor_open(txn, o_txs, &c_txs); if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); - - /* we don't care about the unlocks/heights content or cursor position. - * we just delete 1 record from each whenever we add a record. - */ - mdb_cursor_get(c_tx_unlocks, &k, &v, MDB_FIRST); - mdb_cursor_get(c_tx_heights, &k, &v, MDB_FIRST); - + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_tx_indices, &ms); - i = ms.ms_entries; + mdb_stat(txn, m_output_txs, &ms); + m_num_outputs = ms.ms_entries; + mdb_stat(txn, m_txs, &ms); + m_num_txs = i = ms.ms_entries; if (i) { - /* remember the ID of the last block we migrated */ + m_num_txs = i; MDB_val_set(pk, "txblk"); - result = mdb_cursor_get(c_props, &pk, &v, MDB_SET); + result = mdb_cursor_get(c_props, &pk, &k, MDB_SET); if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_indices: ", result).c_str())); - ti.data.block_id = *(uint64_t *)v.mv_data; + throw0(DB_ERROR(lmdb_error("Failed to get a record from properties: ", result).c_str())); + m_height = *(uint64_t *)k.mv_data; } - num_txs = i; } if (i) { - k.mv_data = (void *)&ti.data.block_id; - k.mv_size = sizeof(ti.data.block_id); result = mdb_cursor_get(c_blocks, &k, &v, MDB_SET); if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str())); @@ -3459,223 +3265,32 @@ void BlockchainLMDB::migrate_0_1() MDB_val_set(pk, "txblk"); mdb_cursor_get(c_props, &pk, &v, MDB_SET); mdb_cursor_del(c_props, 0); - txn.commit(); + batch_stop(); break; } else if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str())); + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); if (!parse_and_validate_block_from_blob(bd, b)) throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); - ti.key = get_transaction_hash(b.miner_tx); - ti.data.tx_id = num_txs++; - ti.data.block_id = *(uint64_t *)k.mv_data; - ti.data.unlock_time = b.miner_tx.unlock_time; - i++; - - result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &iv, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_indices: ", result).c_str())); - - result = mdb_cursor_del(c_tx_unlocks, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_unlocks: ", result).c_str())); - result = mdb_cursor_del(c_tx_heights, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_heights: ", result).c_str())); - - for (j=0; j<b.tx_hashes.size(); j++) { - ti.key = b.tx_hashes[j]; - ti.data.tx_id = num_txs++; - result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &iv, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_indices: ", result).c_str())); - result = mdb_cursor_del(c_tx_unlocks, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_unlocks: ", result).c_str())); - result = mdb_cursor_del(c_tx_heights, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_heights: ", result).c_str())); - i++; - } - } - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_drop(txn, o_tx_unlocks, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete tx_unlocks from the db: ", result).c_str())); - result = mdb_drop(txn, o_tx_heights, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete tx_heights from the db: ", result).c_str())); - txn.commit(); - m_num_txs = num_txs; - } while(0); - - do { - LOG_PRINT_L1("migrating txs and tx_outputs:"); - - unsigned int flags; - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_dbi_flags(txn, m_txs, &flags); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to retrieve txs flags: ", result).c_str())); - /* if the flags are what we expect, this table has already been migrated */ - if (flags & MDB_INTEGERKEY) { - txn.abort(); - LOG_PRINT_L1(" txs already migrated"); - break; - } - - MDB_dbi o_txs, o_tx_outputs; - txindex *txp; - MDB_val ik, iv; - - o_txs = m_txs; - lmdb_db_open(txn, "txr", MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for txr"); - o_tx_outputs = m_tx_outputs; - lmdb_db_open(txn, "tx_outputr", MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for tx_outputr"); - mdb_set_dupsort(txn, o_tx_outputs, compare_uint64); - - /* a hack to let us find amount index from global index */ - mdb_set_dupsort(txn, m_output_amounts, compare_amtidx); - - { - MDB_stat ms; - mdb_stat(txn, m_tx_indices, &ms); - m_num_txs = ms.ms_entries; - } - - MDB_cursor *c_cur, *c_curtxo, *c_txi; - MDB_cursor *c_txs, *c_tx_outputs; - MDB_cursor *c_oamts; - i = 0; - z = m_num_txs; - - while(1) { - if (!(i % 1000)) { - if (i) { - LOGIF(1) { - std::cout << i << " / " << z << " \r" << std::flush; - } - txn.commit(); - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - } - result = mdb_cursor_open(txn, m_txs, &c_cur); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txr: ", result).c_str())); - result = mdb_cursor_open(txn, o_txs, &c_txs); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str())); - result = mdb_cursor_open(txn, o_tx_outputs, &c_tx_outputs); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_outputs: ", result).c_str())); - result = mdb_cursor_open(txn, m_tx_outputs, &c_curtxo); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_outputr: ", result).c_str())); - result = mdb_cursor_open(txn, m_tx_indices, &c_txi); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); - result = mdb_cursor_open(txn, m_output_amounts, &c_oamts); + add_transaction(null_hash, b.miner_tx); + for (unsigned int j = 0; j<b.tx_hashes.size(); j++) { + transaction tx; + hk.mv_data = &b.tx_hashes[j]; + result = mdb_cursor_get(c_txs, &hk, &v, MDB_SET); if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for output_amounts: ", result).c_str())); - if (!i) { - MDB_stat ms; - mdb_stat(txn, m_txs, &ms); - i = ms.ms_entries; - } - if (i) { - result = mdb_cursor_get(c_txs, &k, &v, MDB_FIRST); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str())); - result = mdb_cursor_get(c_txi, (MDB_val *)&zerokval, &k, MDB_GET_BOTH); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_indices: ", result).c_str())); - result = mdb_cursor_get(c_txi, &k, &v, MDB_PREV); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_indices: ", result).c_str())); - } - } - /* The new tx_indices and the old txs tables are both sorted using - * compare_hash32 so we can walk thru them sequentially and they will - * stay in lock step with each other. Unfortunately, even though the - * old tx_outputs was keyed with the tx hash, it wasn't set to use - * the compare_hash32 function, so its records are in some other random - * hash order. This costs us the majority of CPU time when walking - * thru that table. Iterating with MDB_NEXT is cheap/free, but searching - * through a multi-million record table with 32byte keys is quite costly. - */ - result = mdb_cursor_get(c_txi, &k, &v, MDB_NEXT); - if (result == MDB_NOTFOUND) { - txn.commit(); - break; - } else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_indices: ", result).c_str())); - txp = (txindex *)v.mv_data; - k.mv_data = (void *)&txp->key; - k.mv_size = sizeof(txp->key); - ik.mv_data = (void *)&txp->data.tx_id; - ik.mv_size = sizeof(txp->data.tx_id); - - result = mdb_cursor_get(c_txs, &k, &v, MDB_FIRST); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str())); - result = mdb_cursor_put(c_cur, &ik, &v, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into txr: ", result).c_str())); - - result = mdb_cursor_get(c_tx_outputs, &k, &iv, MDB_SET); - if (!result) { - blobdata bd; + throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); - - transaction tx; if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - - /* turn global output indices into amount output indices */ - std::vector<uint64_t> indices; - int j; - for (j=0;;j++) { - MDB_val_set(vk, tx.vout[j].amount); - result = mdb_cursor_get(c_oamts, &vk, &iv, MDB_GET_BOTH); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from output_amounts: ", result).c_str())); - - outkey *ok = (outkey *)v.mv_data; - indices.push_back(ok->amount_index); - result = mdb_cursor_get(c_tx_outputs, &k, &iv, MDB_NEXT_DUP); - if (result == MDB_NOTFOUND) - break; - else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_outputs: ", result).c_str())); - } - v.mv_data = (void *)indices.data(); - v.mv_size = sizeof(uint64_t) * indices.size(); - result = mdb_cursor_put(c_curtxo, &ik, &v, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_outputr: ", result).c_str())); - result = mdb_cursor_del(c_tx_outputs, MDB_NODUPDATA); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from tx_outputs: ", result).c_str())); - } else if (result == MDB_NOTFOUND) { - /* get_tx_amount_output_indices expects a record here, even if it's empty */ - v.mv_size = 0; - v.mv_data = NULL; - result = mdb_cursor_put(c_curtxo, &ik, &v, 0); + add_transaction(null_hash, tx, &b.tx_hashes[j]); + result = mdb_cursor_del(c_txs, 0); if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into tx_outputr: ", result).c_str())); - } else - throw0(DB_ERROR(lmdb_error("Failed to get a record from tx_outputs: ", result).c_str())); - - result = mdb_cursor_del(c_txs, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from txs: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); + } i++; + m_height = i; } result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) @@ -3683,103 +3298,12 @@ void BlockchainLMDB::migrate_0_1() result = mdb_drop(txn, o_txs, 1); if (result) throw0(DB_ERROR(lmdb_error("Failed to delete txs from the db: ", result).c_str())); - result = mdb_drop(txn, o_tx_outputs, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete tx_outputs from the db: ", result).c_str())); RENAME_DB("txr"); - RENAME_DB("tx_outputr"); mdb_dbi_close(m_env, m_txs); - mdb_dbi_close(m_env, m_tx_outputs); lmdb_db_open(txn, "txs", MDB_INTEGERKEY, m_txs, "Failed to open db handle for txs"); - lmdb_db_open(txn, "tx_outputs", MDB_INTEGERKEY, m_tx_outputs, "Failed to open db handle for tx_outputs"); - mdb_set_dupsort(txn, m_output_amounts, compare_uint64); - - txn.commit(); - } while(0); - - do { - LOG_PRINT_L1("migrating spent_keys:"); - MDB_dbi o_keys; - - unsigned int flags; - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - result = mdb_dbi_flags(txn, m_spent_keys, &flags); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to retrieve spent_keys flags: ", result).c_str())); - /* if the flags are what we expect, this table has already been migrated */ - if ((flags & (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) == (MDB_INTEGERKEY|MDB_DUPSORT|MDB_DUPFIXED)) { - txn.abort(); - LOG_PRINT_L1(" spent_keys already migrated"); - break; - } - - /* the spent_keys table name is the same but the old version and new version - * have incompatible DB flags. Create a new table with the right flags. - */ - o_keys = m_spent_keys; - lmdb_db_open(txn, "spent_keyr", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for spent_keyr"); - mdb_set_dupsort(txn, m_spent_keys, compare_hash32); - - MDB_cursor *c_old, *c_cur; - i = 0; - MDB_stat ms; - mdb_stat(txn, o_keys, &ms); - z = ms.ms_entries; - mdb_stat(txn, m_spent_keys, &ms); - z += ms.ms_entries; - - while(1) { - if (!(i % 2000)) { - if (i) { - LOGIF(1) { - std::cout << i << " / " << z << " \r" << std::flush; - } - txn.commit(); - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - } - result = mdb_cursor_open(txn, m_spent_keys, &c_cur); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keyr: ", result).c_str())); - result = mdb_cursor_open(txn, o_keys, &c_old); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keys: ", result).c_str())); - if (!i) - i = ms.ms_entries; - } - result = mdb_cursor_get(c_old, &k, NULL, MDB_NEXT); - if (result == MDB_NOTFOUND) { - txn.commit(); - break; - } - else if (result) - throw0(DB_ERROR(lmdb_error("Failed to get a record from spent_keys: ", result).c_str())); - result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &k, MDB_APPENDDUP); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to put a record into spent_keyr: ", result).c_str())); - result = mdb_cursor_del(c_old, 0); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete a record from spent_keys: ", result).c_str())); - i++; - } - - result = mdb_txn_begin(m_env, NULL, 0, txn); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); - /* Delete the old table */ - result = mdb_drop(txn, o_keys, 1); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to delete old spent_keys table: ", result).c_str())); - RENAME_DB("spent_keyr"); - mdb_dbi_close(m_env, m_spent_keys); - lmdb_db_open(txn, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for spent_keys"); - mdb_set_dupsort(txn, m_spent_keys, compare_hash32); txn.commit(); } while(0); |