aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--contrib/epee/include/net/abstract_tcp_server2.inl2
-rw-r--r--external/db_drivers/liblmdb/lmdb.h19
-rw-r--r--external/db_drivers/liblmdb/mdb.c80
-rw-r--r--src/blockchain_db/lmdb/db_lmdb.cpp22
-rw-r--r--src/blockchain_db/lmdb/db_lmdb.h2
-rw-r--r--tests/core_tests/chaingen.h2
6 files changed, 99 insertions, 28 deletions
diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl
index 934132ea2..4fb6fa75d 100644
--- a/contrib/epee/include/net/abstract_tcp_server2.inl
+++ b/contrib/epee/include/net/abstract_tcp_server2.inl
@@ -144,7 +144,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
// create a random uuid
boost::uuids::uuid random_uuid;
// that stuff turns out to be included, even though it's from src... Taking advantage
- crypto::generate_random_bytes(sizeof(random_uuid), &random_uuid);
+ random_uuid = crypto::rand<boost::uuids::uuid>();
context.set_details(random_uuid, ip_, remote_ep.port(), is_income);
_dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
diff --git a/external/db_drivers/liblmdb/lmdb.h b/external/db_drivers/liblmdb/lmdb.h
index d616cabf9..a794c9f7d 100644
--- a/external/db_drivers/liblmdb/lmdb.h
+++ b/external/db_drivers/liblmdb/lmdb.h
@@ -78,6 +78,11 @@
* access to locks and lock file. Exceptions: On read-only filesystems
* or with the #MDB_NOLOCK flag described under #mdb_env_open().
*
+ * - An LMDB configuration will often reserve considerable \b unused
+ * memory address space and maybe file size for future growth.
+ * This does not use actual memory or disk space, but users may need
+ * to understand the difference so they won't be scared off.
+ *
* - By default, in versions before 0.9.10, unused portions of the data
* file might receive garbage data from memory freed by other code.
* (This does not happen when using the #MDB_WRITEMAP flag.) As of
@@ -130,7 +135,7 @@
*
* @author Howard Chu, Symas Corporation.
*
- * @copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
+ * @copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
@@ -392,7 +397,9 @@ typedef enum MDB_cursor_op {
MDB_PREV_NODUP, /**< Position at last data item of previous key */
MDB_SET, /**< Position at specified key */
MDB_SET_KEY, /**< Position at specified key, return key + data */
- MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */
+ MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */
+ MDB_PREV_MULTIPLE /**< Position at previous page and return key and up to
+ a page of duplicate data items. Only for #MDB_DUPFIXED */
} MDB_cursor_op;
/** @defgroup errors Return Codes
@@ -538,9 +545,11 @@ int mdb_env_create(MDB_env **env);
* allowed. LMDB will still modify the lock file - except on read-only
* filesystems, where LMDB does not use locks.
* <li>#MDB_WRITEMAP
- * Use a writeable memory map unless MDB_RDONLY is set. This is faster
- * and uses fewer mallocs, but loses protection from application bugs
+ * Use a writeable memory map unless MDB_RDONLY is set. This uses
+ * fewer mallocs but loses protection from application bugs
* like wild pointer writes and other bad updates into the database.
+ * This may be slightly faster for DBs that fit entirely in RAM, but
+ * is slower for DBs larger than RAM.
* Incompatible with nested transactions.
* Do not mix processes with and without MDB_WRITEMAP on the same
* environment. This can defeat durability (#mdb_env_sync etc).
@@ -1535,7 +1544,7 @@ int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags);
* <li>EINVAL - cursor is not initialized, or an invalid parameter was specified.
* </ul>
*/
-int mdb_cursor_count(MDB_cursor *cursor, size_t *countp);
+int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp);
/** @brief Compare two data items according to a particular database.
*
diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c
index 7be13997a..a366b4024 100644
--- a/external/db_drivers/liblmdb/mdb.c
+++ b/external/db_drivers/liblmdb/mdb.c
@@ -5,7 +5,7 @@
* BerkeleyDB API, but much simplified.
*/
/*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -1075,7 +1075,7 @@ typedef struct MDB_db {
pgno_t md_branch_pages; /**< number of internal pages */
pgno_t md_leaf_pages; /**< number of leaf pages */
pgno_t md_overflow_pages; /**< number of overflow pages */
- pgno_t md_entries; /**< number of data items */
+ mdb_size_t md_entries; /**< number of data items */
pgno_t md_root; /**< the root page of this tree */
} MDB_db;
@@ -1426,7 +1426,7 @@ typedef struct MDB_ntxn {
#endif
/** max bytes to write in one call */
-#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4))
+#define MAX_WRITE (0x40000000U >> (sizeof(ssize_t) == 4))
/** Check \b txn and \b dbi arguments to a function */
#define TXN_DBI_EXIST(txn, dbi, validity) \
@@ -4160,6 +4160,19 @@ mdb_env_create(MDB_env **env)
return MDB_SUCCESS;
}
+#ifdef _WIN32
+/** @brief Map a result from an NTAPI call to WIN32. */
+static DWORD
+mdb_nt2win32(NTSTATUS st)
+{
+ OVERLAPPED o = {0};
+ DWORD br;
+ o.Internal = st;
+ GetOverlappedResult(NULL, &o, &br, FALSE);
+ return GetLastError();
+}
+#endif
+
static int ESECT
mdb_env_map(MDB_env *env, void *addr)
{
@@ -4189,7 +4202,7 @@ mdb_env_map(MDB_env *env, void *addr)
rc = NtCreateSection(&mh, access, NULL, NULL, secprot, SEC_RESERVE, env->me_fd);
if (rc)
- return rc;
+ return mdb_nt2win32(rc);
map = addr;
#ifdef MDB_VL32
msize = NUM_METAS * env->me_psize;
@@ -4201,7 +4214,7 @@ mdb_env_map(MDB_env *env, void *addr)
NtClose(mh);
#endif
if (rc)
- return rc;
+ return mdb_nt2win32(rc);
env->me_map = map;
#else
#ifdef MDB_VL32
@@ -5256,14 +5269,16 @@ mdb_env_close0(MDB_env *env, int excl)
free(env->me_dbflags);
free(env->me_path);
free(env->me_dirty_list);
- free(env->me_txn0);
#ifdef MDB_VL32
+ if (env->me_txn0 && env->me_txn0->mt_rpages)
+ free(env->me_txn0->mt_rpages);
{ unsigned int x;
for (x=1; x<=env->me_rpages[0].mid; x++)
munmap(env->me_rpages[x].mptr, env->me_rpages[x].mcnt * env->me_psize);
}
free(env->me_rpages);
#endif
+ free(env->me_txn0);
mdb_midl_free(env->me_free_pgs);
if (env->me_flags & MDB_ENV_TXKEY) {
@@ -5691,7 +5706,8 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret)
#define MAP(rc,env,addr,len,off) \
addr = NULL; \
rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \
- len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY)
+ len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \
+ if (rc) rc = mdb_nt2win32(rc)
#else
off_t off;
size_t len;
@@ -5765,7 +5781,7 @@ notlocal:
pthread_mutex_lock(&env->me_rpmutex);
retry:
y = 0;
- for (i=1; i<tl[0].mid; i++) {
+ for (i=1; i<=tl[0].mid; i++) {
if (!tl[i].mref) {
if (!y) y = i;
/* tmp overflow pages don't go to env */
@@ -5845,7 +5861,7 @@ retry:
if (el[0].mid >= MDB_ERPAGE_MAX - env->me_rpcheck) {
/* purge unref'd pages */
unsigned i, y = 0;
- for (i=1; i<el[0].mid; i++) {
+ for (i=1; i<=el[0].mid; i++) {
if (!el[i].mref) {
if (!y) y = i;
munmap(el[i].mptr, env->me_psize * el[i].mcnt);
@@ -6436,8 +6452,10 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
DPRINTF(("cursor_next: top page is %"Y"u in cursor %p",
mdb_dbg_pgno(mp), (void *) mc));
- if (mc->mc_flags & C_DEL)
+ if (mc->mc_flags & C_DEL) {
+ mc->mc_flags ^= C_DEL;
goto skip;
+ }
if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) {
DPUTS("=====> move to next sibling page");
@@ -6523,6 +6541,8 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
DPRINTF(("cursor_prev: top page is %"Y"u in cursor %p",
mdb_dbg_pgno(mp), (void *) mc));
+ mc->mc_flags &= ~(C_EOF|C_DEL);
+
if (mc->mc_ki[mc->mc_top] == 0) {
DPUTS("=====> move to prev sibling page");
if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) {
@@ -6977,6 +6997,30 @@ fetchm:
}
}
break;
+ case MDB_PREV_MULTIPLE:
+ if (data == NULL) {
+ rc = EINVAL;
+ break;
+ }
+ if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) {
+ rc = MDB_INCOMPATIBLE;
+ break;
+ }
+ if (!(mc->mc_flags & C_INITIALIZED))
+ rc = mdb_cursor_last(mc, key, data);
+ else
+ rc = MDB_SUCCESS;
+ if (rc == MDB_SUCCESS) {
+ MDB_cursor *mx = &mc->mc_xcursor->mx_cursor;
+ if (mx->mc_flags & C_INITIALIZED) {
+ rc = mdb_cursor_sibling(mx, 0);
+ if (rc == MDB_SUCCESS)
+ goto fetchm;
+ } else {
+ rc = MDB_NOTFOUND;
+ }
+ }
+ break;
case MDB_NEXT:
case MDB_NEXT_DUP:
case MDB_NEXT_NODUP:
@@ -7518,7 +7562,7 @@ new_sub:
*/
if (do_sub) {
int xflags, new_dupdata;
- size_t ecount;
+ mdb_size_t ecount;
put_sub:
xdata.mv_size = 0;
xdata.mv_data = "";
@@ -8233,7 +8277,7 @@ mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc)
/* Return the count of duplicate data items for the current key */
int
-mdb_cursor_count(MDB_cursor *mc, size_t *countp)
+mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp)
{
MDB_node *leaf;
@@ -9957,7 +10001,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
MDB_txn *txn = NULL;
mdb_mutexref_t wmutex = NULL;
int rc;
- size_t wsize;
+ mdb_size_t wsize, w3;
char *ptr;
#ifdef _WIN32
DWORD len, w2;
@@ -10016,15 +10060,15 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
if (rc)
goto leave;
- w2 = txn->mt_next_pgno * env->me_psize;
+ w3 = txn->mt_next_pgno * env->me_psize;
{
mdb_size_t fsize = 0;
if ((rc = mdb_fsize(env->me_fd, &fsize)))
goto leave;
- if (w2 > fsize)
- w2 = fsize;
+ if (w3 > fsize)
+ w3 = fsize;
}
- wsize = w2 - wsize;
+ wsize = w3 - wsize;
while (wsize > 0) {
if (wsize > MAX_WRITE)
w2 = MAX_WRITE;
@@ -10093,7 +10137,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
#ifdef _WIN32
rc = utf8_to_utf16(lpath, -1, &wpath, NULL);
if (rc)
- return rc;
+ goto leave;
newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL);
free(wpath);
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index 0bee8aae9..a93079aa7 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -428,7 +428,7 @@ void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size");
- const uint64_t min_increase_size = 128 * (1 << 20);
+ const uint64_t min_increase_size = 512 * (1 << 20);
uint64_t threshold_size = 0;
uint64_t increase_size = 0;
if (batch_num_blocks > 0)
@@ -464,6 +464,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
// batch size estimate * batch safety factor = final size estimate
// Takes into account "reasonable" block size increases in batch.
float batch_safety_factor = 1.7f;
+ float batch_fudge_factor = batch_safety_factor * batch_num_blocks;
// estimate of stored block expanded from raw block, including denormalization and db overhead.
// Note that this probably doesn't grow linearly with block size.
float db_expand_factor = 4.5f;
@@ -485,6 +486,13 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
{
LOG_PRINT_L1("No existing blocks to check for average block size");
}
+ else if (m_cum_count)
+ {
+ avg_block_size = m_cum_size / m_cum_count;
+ LOG_PRINT_L1("average block size across recent " << m_cum_count << " blocks: " << avg_block_size);
+ m_cum_size = 0;
+ m_cum_count = 0;
+ }
else
{
for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
@@ -502,8 +510,10 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
avg_block_size = min_block_size;
LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size);
- threshold_size = avg_block_size * db_expand_factor * batch_num_blocks;
- threshold_size = threshold_size * batch_safety_factor;
+ // bigger safety margin on smaller block sizes
+ if (batch_fudge_factor < 5000.0)
+ batch_fudge_factor = 5000.0;
+ threshold_size = avg_block_size * db_expand_factor * batch_fudge_factor;
return threshold_size;
}
@@ -570,6 +580,8 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
if (result)
throw0(DB_ERROR(std::string("Failed to add block hash to db transaction: ").append(mdb_strerror(result)).c_str()));
+ m_cum_size += block_size;
+ m_cum_count++;
}
void BlockchainLMDB::remove_block()
@@ -945,6 +957,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
m_write_batch_txn = nullptr;
m_batch_active = false;
m_height = 0;
+ m_cum_size = 0;
+ m_cum_count = 0;
m_hardfork = nullptr;
}
@@ -1208,6 +1222,8 @@ void BlockchainLMDB::reset()
txn.commit();
m_height = 0;
m_num_outputs = 0;
+ m_cum_size = 0;
+ m_cum_count = 0;
}
std::vector<std::string> BlockchainLMDB::get_filenames() const
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index e88bcd01b..1964bfe45 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -306,6 +306,8 @@ private:
uint64_t m_height;
uint64_t m_num_outputs;
+ mutable uint64_t m_cum_size; // used in batch size estimation
+ mutable int m_cum_count;
std::string m_folder;
mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h
index f2bbb7346..d0d912cbb 100644
--- a/tests/core_tests/chaingen.h
+++ b/tests/core_tests/chaingen.h
@@ -497,7 +497,7 @@ inline bool do_replay_events(std::vector<test_event_entry>& events)
cryptonote::core c(&pr);
// FIXME: make sure that vm has arg_testnet_on set to true or false if
// this test needs for it to be so.
- const get_test_options<t_test_class> gto;
+ get_test_options<t_test_class> gto;
if (!c.init(vm, &gto.test_options))
{
std::cout << concolor::magenta << "Failed to init core" << concolor::normal << std::endl;
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-05 20:32:20 +0000